Commit d5e069c2 authored by Lucian Grijincu's avatar Lucian Grijincu Committed by Facebook Github Bot

folly: add support for MLOCK_ONFAULT

Summary:
http://man7.org/linux/man-pages/man2/mlock.2.html

       MLOCK_ONFAULT
              Lock pages that are currently resident and mark the entire
              range so that the remaining nonresident pages are locked when
              they are populated by a page fault.

The kernel still checks a few things for `MLOCK_ONFAULT`: rlimits RLIMIT_MEMLOCK / CAP_IPC_LOCK => EPERM or ENOMEM

`TRY_LOCK` & `MUST_LOCK` are only different in in their handling of EPERM, ENOMEN: whether to fail or ignore the errors, so I added a separate enum.

https://github.com/torvalds/linux/blob/v5.0/mm/mlock.c#L29-L37
```
bool can_do_mlock(void)
{
	if (rlimit(RLIMIT_MEMLOCK) != 0)
		return true;
	if (capable(CAP_IPC_LOCK))
		return true;
	return false;
}
EXPORT_SYMBOL(can_do_mlock);
```

https://github.com/torvalds/linux/blob/v5.0/mm/mlock.c#L671-L714
```
static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
{
	unsigned long locked;
	unsigned long lock_limit;
	int error = -ENOMEM;

	if (!can_do_mlock())
		return -EPERM;

	len = PAGE_ALIGN(len + (offset_in_page(start)));
	start &= PAGE_MASK;

	lock_limit = rlimit(RLIMIT_MEMLOCK);
	lock_limit >>= PAGE_SHIFT;
	locked = len >> PAGE_SHIFT;

	if (down_write_killable(&current->mm->mmap_sem))
		return -EINTR;

	locked += current->mm->locked_vm;
	if ((locked > lock_limit) && (!capable(CAP_IPC_LOCK))) {
		/*
		 * It is possible that the regions requested intersect with
		 * previously mlocked areas, that part area in "mm->locked_vm"
		 * should not be counted to new mlock increment count. So check
		 * and adjust locked count if necessary.
		 */
		locked -= count_mm_mlocked_page_nr(current->mm,
				start, len);
	}

	/* check against resource limits */
	if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
		error = apply_vma_lock_flags(start, len, flags);

	up_write(&current->mm->mmap_sem);
	if (error)
		return error;

	error = __mm_populate(start, len, 0);
	if (error)
		return __mlock_posix_error_return(error);
	return 0;
}
```

https://github.com/torvalds/linux/blob/v5.0/mm/mlock.c#L721-L732
```
SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags)
{
	vm_flags_t vm_flags = VM_LOCKED;

	if (flags & ~MLOCK_ONFAULT)
		return -EINVAL;

	if (flags & MLOCK_ONFAULT)
		vm_flags |= VM_LOCKONFAULT;

	return do_mlock(start, len, vm_flags);
}

```

Reviewed By: yfeldblum

Differential Revision: D16760748

fbshipit-source-id: 54937046f5faaceac2cfb9dd6a0f207ad653fe6b
parent 619679d3
......@@ -17,6 +17,7 @@
#include <folly/system/MemoryMapping.h>
#include <algorithm>
#include <cerrno>
#include <functional>
#include <utility>
......@@ -25,6 +26,7 @@
#include <folly/Format.h>
#include <folly/portability/GFlags.h>
#include <folly/portability/SysMman.h>
#include <folly/portability/SysSyscall.h>
#ifdef __linux__
#include <folly/experimental/io/HugePages.h> // @manual
......@@ -266,12 +268,40 @@ bool memOpInChunks(
return true;
}
/**
* mlock2 is Linux-only and exists since Linux 4.4
* On Linux pre-4.4 and other platforms fail with ENOSYS.
* glibc added the mlock2 wrapper in 2.27
* https://lists.gnu.org/archive/html/info-gnu/2018-02/msg00000.html
*/
int mlock2wrapper(const void* addr, size_t len, int flags) {
#if defined(__GLIBC__) && !defined(__APPLE__)
#if __GLIBC_PREREQ(2, 27)
return mlock2(addr, len, flags);
#elif defined(SYS_mlock2)
// SYS_mlock2 is defined in Linux headers since 4.4
return syscall(SYS_mlock2, addr, len, flags);
#else // !__GLIBC_PREREQ(2, 27) && !defined(SYS_mlock2)
errno = ENOSYS;
return -1;
#endif
#else // !defined(__GLIBC__) || defined(__APPLE__)
errno = ENOSYS;
return -1;
#endif
}
} // namespace
bool MemoryMapping::mlock(LockMode lock) {
bool MemoryMapping::mlock(LockMode mode, LockFlags flags) {
size_t amountSucceeded = 0;
locked_ = memOpInChunks(
::mlock,
[flags](void* addr, size_t len) -> int {
// If flags is 0, mlock2() behaves exactly the same as mlock().
// Prefer the portable variant.
return int(flags) == 0 ? ::mlock(addr, len)
: mlock2wrapper(addr, len, int(flags));
},
mapStart_,
size_t(mapLength_),
options_.pageSize,
......@@ -282,9 +312,9 @@ bool MemoryMapping::mlock(LockMode lock) {
auto msg =
folly::format("mlock({}) failed at {}", mapLength_, amountSucceeded);
if (lock == LockMode::TRY_LOCK && errno == EPERM) {
if (mode == LockMode::TRY_LOCK && errno == EPERM) {
PLOG(WARNING) << msg;
} else if (lock == LockMode::TRY_LOCK && errno == ENOMEM) {
} else if (mode == LockMode::TRY_LOCK && errno == ENOMEM) {
VLOG(1) << msg;
} else {
PLOG(FATAL) << msg;
......@@ -426,4 +456,10 @@ void mmapFileCopy(const char* src, const char* dest, mode_t mode) {
srcMap.range().size());
}
MemoryMapping::LockFlags operator|(
MemoryMapping::LockFlags a,
MemoryMapping::LockFlags b) {
return MemoryMapping::LockFlags(int(a) | int(b));
}
} // namespace folly
......@@ -23,6 +23,7 @@
namespace folly {
/**
* Maps files in memory (read-only).
*
......@@ -39,6 +40,24 @@ class MemoryMapping {
TRY_LOCK,
MUST_LOCK,
};
enum class LockFlags : int {
/**
* All pages that contain a part of the specified address range are
* guaranteed to be resident in RAM when the call returns successfully; the
* pages are guaranteed to stay in RAM until later unlocked.
* Uses mlock or mlock2(flags=0). Portable.
*/
LOCK_PREFAULT = 0,
/**
* Lock pages that are currently resident and mark the entire range to have
* pages locked when they are populated by the page fault.
* Same value as MLOCK_ONFAULT which is defined in a non-portable header.
* Uses mlock2(flags=MLOCK_ONFAULT). Requires Linux >= 4.4.
*/
LOCK_ONFAULT = 0x01, // = np MLOCK_ONFAULT
};
/**
* Map a portion of the file indicated by filename in memory, causing SIGABRT
* on error.
......@@ -159,7 +178,7 @@ class MemoryMapping {
/**
* Lock the pages in memory
*/
bool mlock(LockMode lock);
bool mlock(LockMode mode, LockFlags flags = LockFlags::LOCK_PREFAULT);
/**
* Unlock the pages.
......@@ -269,4 +288,8 @@ void alignedForwardMemcpy(void* dest, const void* src, size_t size);
*/
void mmapFileCopy(const char* src, const char* dest, mode_t mode = 0666);
MemoryMapping::LockFlags operator|(
MemoryMapping::LockFlags a,
MemoryMapping::LockFlags b);
} // namespace folly
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment