10 #include <sys/ioctl.h>
12 #include <sys/types.h>
20 #define SYS_PATH "/sys/devices/system/memory/"
27 static int offline_memory(unsigned long long mem_size_bytes,
28 unsigned long long mem_min_start,
30 unsigned long long *num_bytes,
31 unsigned long long *base_addr);
33 int main(int argc, char * argv[]) {
34 unsigned long long mem_size_bytes = 0;
35 unsigned long long mem_min_start = 0;
41 unsigned long long num_bytes, base_addr;
42 struct v3_mem_region mem;
44 if (argc<2 || argc>8) {
45 printf("usage: v3_mem [-r] [-l] [-n k] [-m n] <memory size (MB)>\n\n"
46 "Allocate memory for use by Palacios.\n\n"
47 "With -k this requests in-kernel allocation.\n"
48 "Without -k this attempts to offline memory via hot remove\n\n"
49 "With -l the request or offlining is limited to first 4 GB\n"
50 "Without -l the request or offlining has no limits\n\n"
51 "With -m n the offline memory search starts at n MB\n"
52 "Without -m n the offline memory search starts at 0 MB\n\n"
53 "With -n i the request is for numa node i\n"
54 "Without -n i the request can be on any numa node\n\n");
58 while ((c=getopt(argc,argv,"klmn:"))!=-1) {
67 mem_min_start = atoll(optarg) * (1024*1024);
74 printf("-n requires the numa node...\n");
76 } else if (optopt=='m') {
77 printf("-m requires the minimum starting address (in MB)...\n");
80 printf("Unknown option %c\n",optopt);
85 printf("Unknown option %c\n",optopt);
90 mem_size_bytes = atoll(argv[optind]) * (1024 * 1024);
92 v3_fd = open(v3_dev, O_RDONLY);
95 printf("Error opening V3Vee control device\n");
100 printf("Trying to offline memory (size=%llu, min_start=%llu, limit32=%d)\n",mem_size_bytes,mem_min_start,limit32);
101 if (offline_memory(mem_size_bytes,mem_min_start,limit32, &num_bytes, &base_addr)) {
102 printf("Could not offline memory\n");
105 mem.type=PREALLOCATED;
107 mem.base_addr=base_addr;
108 mem.num_pages=num_bytes/4096;
110 printf("Generating memory allocation request (size=%llu, limit32=%d)\n", mem_size_bytes, limit32);
111 mem.type = limit32 ? REQUESTED32 : REQUESTED;
114 mem.num_pages = mem_size_bytes / 4096;
117 printf("Allocation request is: type=%d, node=%d, base_addr=0x%llx, num_pages=%llu\n",
118 mem.type, mem.node, mem.base_addr, mem.num_pages);
120 if (ioctl(v3_fd, V3_ADD_MEMORY, &mem)<0) {
121 printf("Request rejected by Palacios\n");
125 printf("Request accepted by Palacios\n");
132 static int dir_filter(const struct dirent * dir) {
133 if (strncmp("memory", dir->d_name, 6) == 0) {
141 static int dir_cmp(const struct dirent **dir1, const struct dirent ** dir2) {
142 int num1 = atoi((*dir1)->d_name + 6);
143 int num2 = atoi((*dir2)->d_name + 6);
150 #define UNWIND(first,last) \
153 for (i = first; i <= last; i++) { \
156 snprintf(name,256,"%smemory%d/state",SYS_PATH,i); \
157 f=fopen(name,"r+"); \
159 perror("Cannot open state file\n"); \
162 printf("Re-onlining block %d (%s)\n",i,name); \
163 fprintf(f,"online\n"); \
169 static int offline_memory(unsigned long long mem_size_bytes,
170 unsigned long long mem_min_start,
172 unsigned long long *num_bytes,
173 unsigned long long *base_addr)
176 unsigned int block_size_bytes = 0;
177 int bitmap_entries = 0;
178 unsigned char * bitmap = NULL;
185 printf("Trying to find %dMB (%d bytes) of memory above %llu with limit32=%d\n", mem_size_bytes/(1024*1024), mem_size_bytes, mem_min_start, limit32);
187 /* Figure out the block size */
190 char tmp_buf[BUF_SIZE];
192 tmp_fd = open(SYS_PATH "block_size_bytes", O_RDONLY);
195 perror("Could not open block size file: " SYS_PATH "block_size_bytes");
199 if (read(tmp_fd, tmp_buf, BUF_SIZE) <= 0) {
200 perror("Could not read block size file: " SYS_PATH "block_size_bytes");
206 block_size_bytes = strtoll(tmp_buf, NULL, 16);
208 printf("Memory block size is %dMB (%d bytes)\n", block_size_bytes / (1024 * 1024), block_size_bytes);
213 num_blocks = mem_size_bytes / block_size_bytes;
214 if (mem_size_bytes % block_size_bytes) num_blocks++;
216 mem_min_start = block_size_bytes *
217 ((mem_min_start / block_size_bytes) + (!!(mem_min_start % block_size_bytes)));
219 printf("Looking for %d blocks of memory starting at %p (block %llu) with limit32=%d\n", num_blocks, (void*)mem_min_start, mem_min_start/block_size_bytes,limit32);
222 // We now need to find <num_blocks> consecutive offlinable memory blocks
224 /* Scan the memory directories */
226 struct dirent ** namelist = NULL;
231 int first_block = mem_min_start/block_size_bytes;
232 int limit_block = 0xffffffff / block_size_bytes; // for 32 bit limiting
234 last_block = scandir(SYS_PATH, &namelist, dir_filter, dir_cmp);
235 bitmap_entries = atoi(namelist[last_block - 1]->d_name + 6) + 1;
237 size = bitmap_entries / 8;
238 if (bitmap_entries % 8) size++;
240 bitmap = alloca(size);
243 printf("ERROR: could not allocate space for bitmap\n");
247 memset(bitmap, 0, size);
249 for (i = 0 ; j < bitmap_entries - 1; i++) {
250 struct dirent * tmp_dir = namelist[i];
252 char status_str[BUF_SIZE];
253 char fname[BUF_SIZE];
255 memset(status_str, 0, BUF_SIZE);
256 memset(fname, 0, BUF_SIZE);
258 snprintf(fname, BUF_SIZE, "%s%s/removable", SYS_PATH, tmp_dir->d_name);
260 j = atoi(tmp_dir->d_name + 6);
266 printf("Skipping %s due to minimum start constraint\n",fname);
270 if (limit32 && i>limit_block) {
271 printf("Skipping %s due to 32 bit constraint\n",fname);
276 // The prospective block must be (a) removable, and (b) currently online
278 printf("Checking %s...", fname);
280 block_fd = open(fname, O_RDONLY);
282 if (block_fd == -1) {
283 printf("Hotpluggable memory not supported or could not determine if block is removable...\n");
287 if (read(block_fd, status_str, BUF_SIZE) <= 0) {
288 perror("Could not read block removability information\n");
292 status_str[BUF_SIZE-1]=0;
296 if (atoi(status_str) == 1) {
297 printf("Removable ");
299 printf("Not removable\n");
303 snprintf(fname, BUF_SIZE, "%s%s/state", SYS_PATH, tmp_dir->d_name);
305 block_fd = open(fname, O_RDONLY);
308 perror("Could not open block state\n");
312 if (read(block_fd, status_str, BUF_SIZE) <=0) {
313 perror("Could not read block state information\n");
317 status_str[BUF_SIZE-1]=0;
321 if (!strncasecmp(status_str,"offline",7)) {
322 printf("and Already Offline (unusable)\n");
323 } else if (!strncasecmp(status_str,"online",6)) {
324 printf("and Online (usable)\n");
325 bitmap[major] |= (0x1 << minor);
327 printf("and in Unknown State '%s' (unusable)\n",status_str);
336 /* Scan bitmap for enough consecutive space */
338 // num_blocks: The number of blocks we need to find
339 // bitmap: bitmap of blocks (1 == allocatable)
340 // bitmap_entries: number of blocks in the system/number of bits in bitmap
341 // reg_start: The block index where our allocation will start
346 for (i = 0; i < bitmap_entries; i++) {
350 if (!(bitmap[i_major] & (0x1 << i_minor))) {
351 reg_start = i + 1; // skip the region start to next entry
358 if (run_len >= num_blocks) {
364 if (run_len < num_blocks) {
365 fprintf(stderr, "Could not find enough consecutive memory blocks... (found %d)\n", run_len);
366 // no offlining yet, so no need to unwind here
372 /* Offline memory blocks starting at reg_start */
376 for (i = 0; i < num_blocks; i++) {
377 FILE * block_file = NULL;
380 memset(fname, 0, 256);
382 snprintf(fname, 256, "%smemory%d/state", SYS_PATH, i + reg_start);
384 block_file = fopen(fname, "r+");
386 if (block_file == NULL) {
387 perror("Could not open block file");
388 UNWIND(reg_start, i+reg_start-1);
393 printf("Offlining block %d (%s)\n", i + reg_start, fname);
394 fprintf(block_file, "offline\n");
402 /* We asked to offline set of blocks, but Linux could have lied.
403 * To be safe, check whether blocks were offlined and start again if not
409 mem_ready = 1; // Hopefully we are ok...
412 for (i = 0; i < num_blocks; i++) {
414 char fname[BUF_SIZE];
415 char status_buf[BUF_SIZE];
418 memset(fname, 0, BUF_SIZE);
419 memset(status_buf, 0, BUF_SIZE);
421 snprintf(fname, BUF_SIZE, "%smemory%d/state", SYS_PATH, i + reg_start);
424 block_fd = open(fname, O_RDONLY);
426 if (block_fd == -1) {
427 perror("Could not open block state file");
431 if (read(block_fd, status_buf, BUF_SIZE) <= 0) {
432 perror("Could not read block state");
436 status_buf[BUF_SIZE]=0;
438 printf("Checking offlined block %d (%s)...", i + reg_start, fname);
440 int ret = strncmp(status_buf, "offline", strlen("offline"));
442 if (ret != 0) { // uh oh
444 int major = (i + reg_start) / 8;
445 int minor = (i + reg_start) % 8;
447 bitmap[major] &= ~(0x1 << minor); // mark the block as not removable in bitmap
449 mem_ready = 0; // Keep searching
451 printf("ERROR - block status is '%s'\n", status_buf);
454 UNWIND(reg_start,reg_start+num_blocks-1);
460 printf("Offlined Memory OK\n");
465 /* Memory is offlined. Calculate size and phys start addr to send to Palacios */
466 *num_bytes = (unsigned long long)(num_blocks) * (unsigned long long)(block_size_bytes);
467 *base_addr = (unsigned long long)(reg_start) * (unsigned long long)(block_size_bytes);