kitten/mm/pmem.c

   1 /* Copyright (c) 2008, Sandia National Laboratories */
   2
   3 #include <lwk/kernel.h>
   4 #include <lwk/spinlock.h>
   5 #include <lwk/string.h>
   6 #include <lwk/list.h>
   7 #include <lwk/log2.h>
   8 #include <lwk/pmem.h>
   9 #include <arch/uaccess.h>
  10
  11 static LIST_HEAD(pmem_list);
  12 static DEFINE_SPINLOCK(pmem_list_lock);
  13
  14 struct pmem_list_entry {
  15         struct list_head        link;
  16         struct pmem_region      rgn;
  17 };
  18
  19 static struct pmem_list_entry *
  20 alloc_pmem_list_entry(void)
  21 {
  22         return kmem_alloc(sizeof(struct pmem_list_entry));
  23 }
  24
  25 static void
  26 free_pmem_list_entry(struct pmem_list_entry *entry)
  27 {
  28         kmem_free(entry);
  29 }
  30
  31 static bool
  32 calc_overlap(const struct pmem_region *a, const struct pmem_region *b,
  33              struct pmem_region *dst)
  34 {
  35         if (!((a->start < b->end) && (a->end > b->start)))
  36                 return false;
  37
  38         if (dst) {
  39                 dst->start = max(a->start, b->start);
  40                 dst->end   = min(a->end, b->end);
  41         }
  42
  43         return true;
  44 }
  45
  46 static bool
  47 regions_overlap(const struct pmem_region *a, const struct pmem_region *b)
  48 {
  49         return calc_overlap(a, b, NULL);
  50 }
  51
  52 static bool
  53 region_is_unique(const struct pmem_region *rgn)
  54 {
  55         struct pmem_list_entry *entry;
  56
  57         list_for_each_entry(entry, &pmem_list, link) {
  58                 if (regions_overlap(rgn, &entry->rgn))
  59                         return false;
  60         }
  61         return true;
  62 }
  63
  64 static bool
  65 region_is_sane(const struct pmem_region *rgn)
  66 {
  67         int i;
  68
  69         if (!rgn)
  70                 return false;
  71
  72         if (rgn->end <= rgn->start)
  73                 return false;
  74
  75         for (i = 0; i < sizeof(rgn->name); i++)  {
  76                 if (rgn->name[i] == '\0')
  77                         break;
  78         }
  79         if (i == sizeof(rgn->name))
  80                 return false;
  81
  82         return true;
  83 }
  84
  85 static bool
  86 region_is_known(const struct pmem_region *rgn)
  87 {
  88         struct pmem_list_entry *entry;
  89         struct pmem_region overlap;
  90         size_t size;
  91
  92         size = rgn->end - rgn->start;
  93         list_for_each_entry(entry, &pmem_list, link) {
  94                 if (!calc_overlap(rgn, &entry->rgn, &overlap))
  95                         continue;
  96
  97                 size -= (overlap.end - overlap.start);
  98         }
  99
 100         return (size == 0) ? true : false;
 101 }
 102
 103 static void
 104 insert_pmem_list_entry(struct pmem_list_entry *entry)
 105 {
 106         struct list_head *pos;
 107         struct pmem_list_entry *cur;
 108
 109         /* Locate the entry that the new entry should be inserted before */
 110         list_for_each(pos, &pmem_list) {
 111                 cur = list_entry(pos, struct pmem_list_entry, link);
 112                 if (cur->rgn.start > entry->rgn.start)
 113                         break;
 114         }
 115         list_add_tail(&entry->link, pos);
 116 }
 117
 118 static bool
 119 regions_are_mergeable(const struct pmem_region *a, const struct pmem_region *b)
 120 {
 121         if ((a->end != b->start) && (b->end != a->start))
 122                 return false;
 123
 124         if (a->type_is_set != b->type_is_set)
 125                 return false;
 126         if (a->type_is_set && (a->type != b->type))
 127                 return false;
 128
 129         if (a->lgroup_is_set != b->lgroup_is_set)
 130                 return false;
 131         if (a->lgroup_is_set && (a->lgroup != b->lgroup))
 132                 return false;
 133
 134         if (a->allocated_is_set != b->allocated_is_set)
 135                 return false;
 136         if (a->allocated_is_set && (a->allocated != b->allocated))
 137                 return false;
 138
 139         if (a->name_is_set != b->name_is_set)
 140                 return false;
 141         if (a->name_is_set && !strcmp(a->name, b->name))
 142                 return false;
 143
 144         return true;
 145 }
 146
 147 static bool
 148 region_matches(const struct pmem_region *query, const struct pmem_region *rgn)
 149 {
 150         if (!regions_overlap(query, rgn))
 151                 return false;
 152
 153         if (query->type_is_set
 154               && (!rgn->type_is_set || (rgn->type != query->type)))
 155                 return false;
 156
 157         if (query->lgroup_is_set
 158               && (!rgn->lgroup_is_set || (rgn->lgroup != query->lgroup)))
 159                 return false;
 160
 161         if (query->allocated_is_set
 162               && (!rgn->allocated_is_set || (rgn->allocated != query->allocated)))
 163                 return false;
 164
 165         if (query->name_is_set
 166               && (!rgn->name_is_set || strcmp(rgn->name, query->name)))
 167                 return false;
 168
 169         return true;
 170 }
 171
 172 static void
 173 merge_pmem_list(void)
 174 {
 175         struct pmem_list_entry *entry, *prev, *tmp;
 176
 177         prev = NULL;
 178         list_for_each_entry_safe(entry, tmp, &pmem_list, link) {
 179                 if (prev && regions_are_mergeable(&prev->rgn, &entry->rgn)) {
 180                         prev->rgn.end = entry->rgn.end;
 181                         list_del(&entry->link);
 182                         free_pmem_list_entry(entry);
 183                 } else {
 184                         prev = entry;
 185                 }
 186         }
 187 }
 188
 189 static void
 190 zero_pmem(const struct pmem_region *rgn)
 191 {
 192         /* access pmem region via the kernel's identity map */
 193         memset(__va(rgn->start), 0, rgn->end - rgn->start);
 194 }
 195
 196 static int
 197 __pmem_add(const struct pmem_region *rgn)
 198 {
 199         struct pmem_list_entry *entry;
 200
 201         if (!region_is_sane(rgn))
 202                 return -EINVAL;
 203
 204         if (!region_is_unique(rgn))
 205                 return -EEXIST;
 206
 207         if (!(entry = alloc_pmem_list_entry()))
 208                 return -ENOMEM;
 209
 210         entry->rgn = *rgn;
 211
 212         insert_pmem_list_entry(entry);
 213         merge_pmem_list();
 214
 215         return 0;
 216 }
 217
 218 int
 219 pmem_add(const struct pmem_region *rgn)
 220 {
 221         int status;
 222         unsigned long irqstate;
 223
 224         spin_lock_irqsave(&pmem_list_lock, irqstate);
 225         status = __pmem_add(rgn);
 226         spin_unlock_irqrestore(&pmem_list_lock, irqstate);
 227
 228         return status;
 229 }
 230
 231 int
 232 sys_pmem_add(const struct pmem_region __user *rgn)
 233 {
 234         struct pmem_region _rgn;
 235
 236         if (current->uid != 0)
 237                 return -EPERM;
 238
 239         if (copy_from_user(&_rgn, rgn, sizeof(_rgn)))
 240                 return -EINVAL;
 241
 242         return pmem_add(&_rgn);
 243 }
 244
 245 static int
 246 __pmem_update(const struct pmem_region *update, bool umem_only)
 247 {
 248         struct pmem_list_entry *entry, *head, *tail;
 249         struct pmem_region overlap;
 250
 251         if (!region_is_sane(update))
 252                 return -EINVAL;
 253
 254         if (!region_is_known(update))
 255                 return -ENOENT;
 256
 257         list_for_each_entry(entry, &pmem_list, link) {
 258                 if (!calc_overlap(update, &entry->rgn, &overlap))
 259                         continue;
 260
 261                 /* Jail user-space to PMEM_TYPE_UMEM regions */
 262                 if (umem_only) {
 263                         if (!entry->rgn.type_is_set
 264                              || (entry->rgn.type != PMEM_TYPE_UMEM))
 265                                 return -EPERM;
 266                         if (!update->type_is_set
 267                              || (update->type != PMEM_TYPE_UMEM))
 268                                 return -EPERM;
 269                 }
 270
 271                 /* Handle head of entry non-overlap */
 272                 if (entry->rgn.start < overlap.start) {
 273                         if (!(head = alloc_pmem_list_entry()))
 274                                 return -ENOMEM;
 275                         head->rgn = entry->rgn;
 276                         head->rgn.end = overlap.start;
 277                         list_add_tail(&head->link, &entry->link);
 278                 }
 279
 280                 /* Handle tail of entry non-overlap */
 281                 if (entry->rgn.end > overlap.end) {
 282                         if (!(tail = alloc_pmem_list_entry()))
 283                                 return -ENOMEM;
 284                         tail->rgn = entry->rgn;
 285                         tail->rgn.start = overlap.end;
 286                         list_add(&tail->link, &entry->link);
 287                 }
 288
 289                 /* Update entry to reflect the overlap */
 290                 entry->rgn = *update;
 291                 entry->rgn.start = overlap.start;
 292                 entry->rgn.end   = overlap.end;
 293         }
 294
 295         merge_pmem_list();
 296
 297         return 0;
 298 }
 299
 300 static int
 301 _pmem_update(const struct pmem_region *update, bool umem_only)
 302 {
 303         int status;
 304         unsigned long irqstate;
 305
 306         spin_lock_irqsave(&pmem_list_lock, irqstate);
 307         status = __pmem_update(update, umem_only);
 308         spin_unlock_irqrestore(&pmem_list_lock, irqstate);
 309
 310         return status;
 311 }
 312
 313 int
 314 pmem_update(const struct pmem_region *update)
 315 {
 316         return _pmem_update(update, false);
 317 }
 318
 319 int
 320 sys_pmem_update(const struct pmem_region __user *update)
 321 {
 322         struct pmem_region _update;
 323
 324         if (current->uid != 0)
 325                 return -EPERM;
 326
 327         if (copy_from_user(&_update, update, sizeof(_update)))
 328                 return -EINVAL;
 329
 330         return _pmem_update(&_update, true);
 331 }
 332
 333 static int
 334 __pmem_query(const struct pmem_region *query, struct pmem_region *result)
 335 {
 336         struct pmem_list_entry *entry;
 337         struct pmem_region *rgn;
 338
 339         if (!region_is_sane(query))
 340                 return -EINVAL;
 341
 342         list_for_each_entry(entry, &pmem_list, link) {
 343                 rgn = &entry->rgn;
 344                 if (!region_matches(query, rgn))
 345                         continue;
 346
 347                 /* match found, update result */
 348                 if (result) {
 349                         *result = *rgn;
 350                         calc_overlap(query, rgn, result);
 351                 }
 352                 return 0;
 353         }
 354
 355         return -ENOENT;
 356 }
 357
 358 int
 359 pmem_query(const struct pmem_region *query, struct pmem_region *result)
 360 {
 361         int status;
 362         unsigned long irqstate;
 363
 364         spin_lock_irqsave(&pmem_list_lock, irqstate);
 365         status = __pmem_query(query, result);
 366         spin_unlock_irqrestore(&pmem_list_lock, irqstate);
 367
 368         return status;
 369 }
 370
 371 int
 372 sys_pmem_query(const struct pmem_region __user *query,
 373                struct pmem_region __user *result)
 374 {
 375         struct pmem_region _query, _result;
 376         int status;
 377
 378         if (current->uid != 0)
 379                 return -EPERM;
 380
 381         if (copy_from_user(&_query, query, sizeof(_query)))
 382                 return -EINVAL;
 383
 384         if ((status = pmem_query(&_query, &_result)) != 0)
 385                 return status;
 386
 387         if (result && copy_to_user(result, &_result, sizeof(*result)))
 388                 return -EINVAL;
 389
 390         return 0;
 391 }
 392
 393 static int
 394 __pmem_alloc(size_t size, size_t alignment,
 395              const struct pmem_region *constraint,
 396              struct pmem_region *result)
 397 {
 398         int status;
 399         struct pmem_region query;
 400         struct pmem_region candidate;
 401
 402         if (size == 0)
 403                 return -EINVAL;
 404
 405         if (alignment && !is_power_of_2(alignment))
 406                 return -EINVAL;
 407
 408         if (!region_is_sane(constraint))
 409                 return -EINVAL;
 410
 411         if (constraint->allocated_is_set && constraint->allocated)
 412                 return -EINVAL;
 413
 414         query = *constraint;
 415
 416         while ((status = __pmem_query(&query, &candidate)) == 0) {
 417                 if (alignment) {
 418                         candidate.start = round_up(candidate.start, alignment);
 419                         if (candidate.start >= candidate.end)
 420                                 continue;
 421                 }
 422
 423                 if ((candidate.end - candidate.start) >= size) {
 424                         candidate.end = candidate.start + size;
 425                         candidate.allocated_is_set = true;
 426                         candidate.allocated = true;
 427                         status = __pmem_update(&candidate, false);
 428                         BUG_ON(status);
 429                         zero_pmem(&candidate);
 430                         if (result)
 431                                 *result = candidate;
 432                         return 0;
 433                 }
 434
 435                 query.start = candidate.end;
 436         }
 437         BUG_ON(status != -ENOENT);
 438
 439         return -ENOMEM;
 440 }
 441
 442 int
 443 pmem_alloc(size_t size, size_t alignment,
 444            const struct pmem_region *constraint,
 445            struct pmem_region *result)
 446 {
 447         int status;
 448         unsigned long irqstate;
 449
 450         spin_lock_irqsave(&pmem_list_lock, irqstate);
 451         status = __pmem_alloc(size, alignment, constraint, result);
 452         spin_unlock_irqrestore(&pmem_list_lock, irqstate);
 453
 454         return status;
 455 }
 456
 457 int
 458 sys_pmem_alloc(size_t size, size_t alignment,
 459                const struct pmem_region __user *constraint,
 460                struct pmem_region __user *result)
 461 {
 462         struct pmem_region _constraint, _result;
 463         int status;
 464
 465         if (current->uid != 0)
 466                 return -EPERM;
 467
 468         if (copy_from_user(&_constraint, constraint, sizeof(_constraint)))
 469                 return -EINVAL;
 470
 471         if ((status = pmem_alloc(size, alignment, &_constraint, &_result)) != 0)
 472                 return status;
 473
 474         if (result && copy_to_user(result, &_result, sizeof(*result)))
 475                 return -EINVAL;
 476
 477         return 0;
 478 }