[RFC 09/16] NOVA: DAX code

[Steven Swanson]

NOVA leverages the kernel's DAX mechanisms for mmap and file data access.  Nova
maintains a red-black tree in DRAM (nova_inode_info_header.vma_tree) to track
which portions of a file have been mapped.

Signed-off-by: Steven Swanson <swanson@xxxxxxxxxxx>
---
 fs/nova/dax.c | 1346 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1346 insertions(+)
 create mode 100644 fs/nova/dax.c

diff --git a/fs/nova/dax.c b/fs/nova/dax.c
new file mode 100644
index 000000000000..871b10f1889c
--- /dev/null
+++ b/fs/nova/dax.c
@@ -0,0 +1,1346 @@
+/*
+ * BRIEF DESCRIPTION
+ *
+ * DAX file operations.
+ *
+ * Copyright 2015-2016 Regents of the University of California,
+ * UCSD Non-Volatile Systems Lab, Andiry Xu <jix024@xxxxxxxxxxx>
+ * Copyright 2012-2013 Intel Corporation
+ * Copyright 2009-2011 Marco Stornelli <marco.stornelli@xxxxxxxxx>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/module.h>
+#include <linux/buffer_head.h>
+#include <linux/cpufeature.h>
+#include <asm/pgtable.h>
+#include <linux/version.h>
+#include "nova.h"
+#include "inode.h"
+
+
+
+static inline int nova_copy_partial_block(struct super_block *sb,
+	struct nova_inode_info_header *sih,
+	struct nova_file_write_entry *entry, unsigned long index,
+	size_t offset, size_t length, void *kmem)
+{
+	void *ptr;
+	int rc = 0;
+	unsigned long nvmm;
+
+	nvmm = get_nvmm(sb, sih, entry, index);
+	ptr = nova_get_block(sb, (nvmm << PAGE_SHIFT));
+
+	if (ptr != NULL) {
+		if (support_clwb)
+			rc = memcpy_mcsafe(kmem + offset, ptr + offset,
+						length);
+		else
+			memcpy_to_pmem_nocache(kmem + offset, ptr + offset,
+						length);
+	}
+
+	/* TODO: If rc < 0, go to MCE data recovery. */
+	return rc;
+}
+
+static inline int nova_handle_partial_block(struct super_block *sb,
+	struct nova_inode_info_header *sih,
+	struct nova_file_write_entry *entry, unsigned long index,
+	size_t offset, size_t length, void *kmem)
+{
+	struct nova_sb_info *sbi = NOVA_SB(sb);
+	struct nova_file_write_entry *entryc, entry_copy;
+
+	nova_memunlock_block(sb, kmem);
+	if (entry == NULL) {
+		/* Fill zero */
+		if (support_clwb)
+			memset(kmem + offset, 0, length);
+		else
+			memcpy_to_pmem_nocache(kmem + offset,
+					sbi->zeroed_page, length);
+	} else {
+		/* Copy from original block */
+		if (metadata_csum == 0)
+			entryc = entry;
+		else {
+			entryc = &entry_copy;
+			if (!nova_verify_entry_csum(sb, entry, entryc))
+				return -EIO;
+		}
+
+		nova_copy_partial_block(sb, sih, entryc, index,
+					offset, length, kmem);
+
+	}
+	nova_memlock_block(sb, kmem);
+	if (support_clwb)
+		nova_flush_buffer(kmem + offset, length, 0);
+	return 0;
+}
+
+/*
+ * Fill the new start/end block from original blocks.
+ * Do nothing if fully covered; copy if original blocks present;
+ * Fill zero otherwise.
+ */
+int nova_handle_head_tail_blocks(struct super_block *sb,
+	struct inode *inode, loff_t pos, size_t count, void *kmem)
+{
+	struct nova_inode_info *si = NOVA_I(inode);
+	struct nova_inode_info_header *sih = &si->header;
+	size_t offset, eblk_offset;
+	unsigned long start_blk, end_blk, num_blocks;
+	struct nova_file_write_entry *entry;
+	timing_t partial_time;
+	int ret = 0;
+
+	NOVA_START_TIMING(partial_block_t, partial_time);
+	offset = pos & (sb->s_blocksize - 1);
+	num_blocks = ((count + offset - 1) >> sb->s_blocksize_bits) + 1;
+	/* offset in the actual block size block */
+	offset = pos & (nova_inode_blk_size(sih) - 1);
+	start_blk = pos >> sb->s_blocksize_bits;
+	end_blk = start_blk + num_blocks - 1;
+
+	nova_dbg_verbose("%s: %lu blocks\n", __func__, num_blocks);
+	/* We avoid zeroing the alloc'd range, which is going to be overwritten
+	 * by this system call anyway
+	 */
+	nova_dbg_verbose("%s: start offset %lu start blk %lu %p\n", __func__,
+				offset, start_blk, kmem);
+	if (offset != 0) {
+		entry = nova_get_write_entry(sb, sih, start_blk);
+		ret = nova_handle_partial_block(sb, sih, entry,
+						start_blk, 0, offset, kmem);
+		if (ret < 0)
+			return ret;
+	}
+
+	kmem = (void *)((char *)kmem +
+			((num_blocks - 1) << sb->s_blocksize_bits));
+	eblk_offset = (pos + count) & (nova_inode_blk_size(sih) - 1);
+	nova_dbg_verbose("%s: end offset %lu, end blk %lu %p\n", __func__,
+				eblk_offset, end_blk, kmem);
+	if (eblk_offset != 0) {
+		entry = nova_get_write_entry(sb, sih, end_blk);
+
+		ret = nova_handle_partial_block(sb, sih, entry, end_blk,
+						eblk_offset,
+						sb->s_blocksize - eblk_offset,
+						kmem);
+		if (ret < 0)
+			return ret;
+	}
+	NOVA_END_TIMING(partial_block_t, partial_time);
+
+	return ret;
+}
+
+int nova_reassign_file_tree(struct super_block *sb,
+	struct nova_inode_info_header *sih, u64 begin_tail)
+{
+	void *addr;
+	struct nova_file_write_entry *entry;
+	struct nova_file_write_entry *entryc, entry_copy;
+	u64 curr_p = begin_tail;
+	size_t entry_size = sizeof(struct nova_file_write_entry);
+
+	entryc = (metadata_csum == 0) ? entry : &entry_copy;
+
+	while (curr_p && curr_p != sih->log_tail) {
+		if (is_last_entry(curr_p, entry_size))
+			curr_p = next_log_page(sb, curr_p);
+
+		if (curr_p == 0) {
+			nova_err(sb, "%s: File inode %lu log is NULL!\n",
+				__func__, sih->ino);
+			return -EINVAL;
+		}
+
+		addr = (void *) nova_get_block(sb, curr_p);
+		entry = (struct nova_file_write_entry *) addr;
+
+		if (metadata_csum == 0)
+			entryc = entry;
+		else if (!nova_verify_entry_csum(sb, entry, entryc))
+			return -EIO;
+
+		if (nova_get_entry_type(entryc) != FILE_WRITE) {
+			nova_dbg("%s: entry type is not write? %d\n",
+				__func__, nova_get_entry_type(entry));
+			curr_p += entry_size;
+			continue;
+		}
+
+		nova_assign_write_entry(sb, sih, entry, entryc, true);
+		curr_p += entry_size;
+	}
+
+	return 0;
+}
+
+int nova_cleanup_incomplete_write(struct super_block *sb,
+	struct nova_inode_info_header *sih, unsigned long blocknr,
+	int allocated, u64 begin_tail, u64 end_tail)
+{
+	void *addr;
+	struct nova_file_write_entry *entry;
+	struct nova_file_write_entry *entryc, entry_copy;
+	u64 curr_p = begin_tail;
+	size_t entry_size = sizeof(struct nova_file_write_entry);
+
+	if (blocknr > 0 && allocated > 0)
+		nova_free_data_blocks(sb, sih, blocknr, allocated);
+
+	if (begin_tail == 0 || end_tail == 0)
+		return 0;
+
+	entryc = (metadata_csum == 0) ? entry : &entry_copy;
+
+	while (curr_p != end_tail) {
+		if (is_last_entry(curr_p, entry_size))
+			curr_p = next_log_page(sb, curr_p);
+
+		if (curr_p == 0) {
+			nova_err(sb, "%s: File inode %lu log is NULL!\n",
+				__func__, sih->ino);
+			return -EINVAL;
+		}
+
+		addr = (void *) nova_get_block(sb, curr_p);
+		entry = (struct nova_file_write_entry *) addr;
+
+		if (metadata_csum == 0)
+			entryc = entry;
+		else {
+			/* skip entry check here as the entry checksum may not
+			 * be updated when this is called
+			 */
+			if (memcpy_mcsafe(entryc, entry,
+					sizeof(struct nova_file_write_entry)))
+				return -EIO;
+		}
+
+		if (nova_get_entry_type(entryc) != FILE_WRITE) {
+			nova_dbg("%s: entry type is not write? %d\n",
+				__func__, nova_get_entry_type(entry));
+			curr_p += entry_size;
+			continue;
+		}
+
+		blocknr = entryc->block >> PAGE_SHIFT;
+		nova_free_data_blocks(sb, sih, blocknr, entryc->num_pages);
+		curr_p += entry_size;
+	}
+
+	return 0;
+}
+
+void nova_init_file_write_entry(struct super_block *sb,
+	struct nova_inode_info_header *sih, struct nova_file_write_entry *entry,
+	u64 epoch_id, u64 pgoff, int num_pages, u64 blocknr, u32 time,
+	u64 file_size)
+{
+	memset(entry, 0, sizeof(struct nova_file_write_entry));
+	entry->entry_type = FILE_WRITE;
+	entry->reassigned = 0;
+	entry->updating = 0;
+	entry->epoch_id = epoch_id;
+	entry->trans_id = sih->trans_id;
+	entry->pgoff = cpu_to_le64(pgoff);
+	entry->num_pages = cpu_to_le32(num_pages);
+	entry->invalid_pages = 0;
+	entry->block = cpu_to_le64(nova_get_block_off(sb, blocknr,
+							sih->i_blk_type));
+	entry->mtime = cpu_to_le32(time);
+
+	entry->size = file_size;
+}
+
+int nova_protect_file_data(struct super_block *sb, struct inode *inode,
+	loff_t pos, size_t count, const char __user *buf, unsigned long blocknr,
+	bool inplace)
+{
+	struct nova_inode_info *si = NOVA_I(inode);
+	struct nova_inode_info_header *sih = &si->header;
+	size_t offset, eblk_offset, bytes, left;
+	unsigned long start_blk, end_blk, num_blocks, nvmm, nvmmoff;
+	unsigned long blocksize = sb->s_blocksize;
+	unsigned int blocksize_bits = sb->s_blocksize_bits;
+	u8 *blockbuf, *blockptr;
+	struct nova_file_write_entry *entry;
+	struct nova_file_write_entry *entryc, entry_copy;
+	bool mapped, nvmm_ok;
+	int ret = 0;
+	timing_t protect_file_data_time, memcpy_time;
+
+	NOVA_START_TIMING(protect_file_data_t, protect_file_data_time);
+
+	offset = pos & (blocksize - 1);
+	num_blocks = ((offset + count - 1) >> blocksize_bits) + 1;
+	start_blk = pos >> blocksize_bits;
+	end_blk = start_blk + num_blocks - 1;
+
+	NOVA_START_TIMING(protect_memcpy_t, memcpy_time);
+	blockbuf = kmalloc(blocksize, GFP_KERNEL);
+	if (blockbuf == NULL) {
+		nova_err(sb, "%s: block buffer allocation error\n", __func__);
+		return -ENOMEM;
+	}
+
+	bytes = blocksize - offset;
+	if (bytes > count)
+		bytes = count;
+
+	left = copy_from_user(blockbuf + offset, buf, bytes);
+	NOVA_END_TIMING(protect_memcpy_t, memcpy_time);
+	if (unlikely(left != 0)) {
+		nova_err(sb, "%s: not all data is copied from user! expect to copy %zu bytes, actually copied %zu bytes\n",
+			 __func__, bytes, bytes - left);
+		ret = -EFAULT;
+		goto out;
+	}
+
+	entryc = (metadata_csum == 0) ? entry : &entry_copy;
+
+	if (offset != 0) {
+		NOVA_STATS_ADD(protect_head, 1);
+		entry = nova_get_write_entry(sb, sih, start_blk);
+		if (entry != NULL) {
+			if (metadata_csum == 0)
+				entryc = entry;
+			else if (!nova_verify_entry_csum(sb, entry, entryc))
+				return -EIO;
+
+			/* make sure data in the partial block head is good */
+			nvmm = get_nvmm(sb, sih, entryc, start_blk);
+			nvmmoff = nova_get_block_off(sb, nvmm, sih->i_blk_type);
+			blockptr = (u8 *) nova_get_block(sb, nvmmoff);
+
+			mapped = nova_find_pgoff_in_vma(inode, start_blk);
+			if (data_csum > 0 && !mapped && !inplace) {
+				nvmm_ok = nova_verify_data_csum(sb, sih, nvmm,
+								0, offset);
+				if (!nvmm_ok) {
+					ret = -EIO;
+					goto out;
+				}
+			}
+
+			ret = memcpy_mcsafe(blockbuf, blockptr, offset);
+			if (ret < 0)
+				goto out;
+		} else {
+			memset(blockbuf, 0, offset);
+		}
+
+		/* copying existing checksums from nvmm can be even slower than
+		 * re-computing checksums of a whole block.
+		if (data_csum > 0)
+			nova_copy_partial_block_csum(sb, sih, entry, start_blk,
+							offset, blocknr, false);
+		*/
+	}
+
+	if (num_blocks == 1)
+		goto eblk;
+
+	do {
+		if (inplace)
+			nova_update_block_csum_parity(sb, sih, blockbuf,
+							blocknr, offset, bytes);
+		else
+			nova_update_block_csum_parity(sb, sih, blockbuf,
+							blocknr, 0, blocksize);
+
+		blocknr++;
+		pos += bytes;
+		buf += bytes;
+		count -= bytes;
+		offset = pos & (blocksize - 1);
+
+		bytes = count < blocksize ? count : blocksize;
+		left = copy_from_user(blockbuf, buf, bytes);
+		if (unlikely(left != 0)) {
+			nova_err(sb, "%s: not all data is copied from user!  expect to copy %zu bytes, actually copied %zu bytes\n",
+				 __func__, bytes, bytes - left);
+			ret = -EFAULT;
+			goto out;
+		}
+	} while (count > blocksize);
+
+eblk:
+	eblk_offset = (pos + count) & (blocksize - 1);
+
+	if (eblk_offset != 0) {
+		NOVA_STATS_ADD(protect_tail, 1);
+		entry = nova_get_write_entry(sb, sih, end_blk);
+		if (entry != NULL) {
+			if (metadata_csum == 0)
+				entryc = entry;
+			else if (!nova_verify_entry_csum(sb, entry, entryc))
+				return -EIO;
+
+			/* make sure data in the partial block tail is good */
+			nvmm = get_nvmm(sb, sih, entryc, end_blk);
+			nvmmoff = nova_get_block_off(sb, nvmm, sih->i_blk_type);
+			blockptr = (u8 *) nova_get_block(sb, nvmmoff);
+
+			mapped = nova_find_pgoff_in_vma(inode, end_blk);
+			if (data_csum > 0 && !mapped && !inplace) {
+				nvmm_ok = nova_verify_data_csum(sb, sih, nvmm,
+					eblk_offset, blocksize - eblk_offset);
+				if (!nvmm_ok) {
+					ret = -EIO;
+					goto out;
+				}
+			}
+
+			ret = memcpy_mcsafe(blockbuf + eblk_offset,
+						blockptr + eblk_offset,
+						blocksize - eblk_offset);
+			if (ret < 0)
+				goto out;
+		} else {
+			memset(blockbuf + eblk_offset, 0,
+				blocksize - eblk_offset);
+		}
+
+		/* copying existing checksums from nvmm can be even slower than
+		 * re-computing checksums of a whole block.
+		if (data_csum > 0)
+			nova_copy_partial_block_csum(sb, sih, entry, end_blk,
+						eblk_offset, blocknr, true);
+		*/
+	}
+
+	if (inplace)
+		nova_update_block_csum_parity(sb, sih, blockbuf, blocknr,
+							offset, bytes);
+	else
+		nova_update_block_csum_parity(sb, sih, blockbuf, blocknr,
+							0, blocksize);
+
+out:
+	if (blockbuf != NULL)
+		kfree(blockbuf);
+
+	NOVA_END_TIMING(protect_file_data_t, protect_file_data_time);
+
+	return ret;
+}
+
+static bool nova_get_verify_entry(struct super_block *sb,
+	struct nova_file_write_entry *entry,
+	struct nova_file_write_entry *entryc,
+	int locked)
+{
+	int ret = 0;
+
+	if (metadata_csum == 0)
+		return true;
+
+	if (locked == 0) {
+		/* Someone else may be updating the entry. Skip check */
+		ret = memcpy_mcsafe(entryc, entry,
+				sizeof(struct nova_file_write_entry));
+		if (ret < 0)
+			return false;
+
+		return true;
+	}
+
+	return nova_verify_entry_csum(sb, entry, entryc);
+}
+
+/*
+ * Check if there is an existing entry for target page offset.
+ * Used for inplace write, direct IO, DAX-mmap and fallocate.
+ */
+unsigned long nova_check_existing_entry(struct super_block *sb,
+	struct inode *inode, unsigned long num_blocks, unsigned long start_blk,
+	struct nova_file_write_entry **ret_entry,
+	struct nova_file_write_entry *ret_entryc, int check_next, u64 epoch_id,
+	int *inplace, int locked)
+{
+	struct nova_inode_info *si = NOVA_I(inode);
+	struct nova_inode_info_header *sih = &si->header;
+	struct nova_file_write_entry *entry;
+	struct nova_file_write_entry *entryc;
+	unsigned long next_pgoff;
+	unsigned long ent_blks = 0;
+	timing_t check_time;
+
+	NOVA_START_TIMING(check_entry_t, check_time);
+
+	*ret_entry = NULL;
+	*inplace = 0;
+	entry = nova_get_write_entry(sb, sih, start_blk);
+
+	entryc = (metadata_csum == 0) ? entry : ret_entryc;
+
+	if (entry) {
+		if (metadata_csum == 0)
+			entryc = entry;
+		else if (!nova_get_verify_entry(sb, entry, entryc, locked))
+			goto out;
+
+		*ret_entry = entry;
+
+		/* We can do inplace write. Find contiguous blocks */
+		if (entryc->reassigned == 0)
+			ent_blks = entryc->num_pages -
+					(start_blk - entryc->pgoff);
+		else
+			ent_blks = 1;
+
+		if (ent_blks > num_blocks)
+			ent_blks = num_blocks;
+
+		if (entryc->epoch_id == epoch_id)
+			*inplace = 1;
+
+	} else if (check_next) {
+		/* Possible Hole */
+		entry = nova_find_next_entry(sb, sih, start_blk);
+		if (entry) {
+			if (metadata_csum == 0)
+				entryc = entry;
+			else if (!nova_get_verify_entry(sb, entry, entryc,
+							locked))
+				goto out;
+
+			next_pgoff = entryc->pgoff;
+			if (next_pgoff <= start_blk) {
+				nova_err(sb, "iblock %lu, entry pgoff %lu, num pages %lu\n",
+				       start_blk, next_pgoff, entry->num_pages);
+				nova_print_inode_log(sb, inode);
+				BUG();
+				ent_blks = num_blocks;
+				goto out;
+			}
+			ent_blks = next_pgoff - start_blk;
+			if (ent_blks > num_blocks)
+				ent_blks = num_blocks;
+		} else {
+			/* File grow */
+			ent_blks = num_blocks;
+		}
+	}
+
+	if (entry && ent_blks == 0) {
+		nova_dbg("%s: %d\n", __func__, check_next);
+		dump_stack();
+	}
+
+out:
+	NOVA_END_TIMING(check_entry_t, check_time);
+	return ent_blks;
+}
+
+ssize_t nova_inplace_file_write(struct file *filp,
+	const char __user *buf,	size_t len, loff_t *ppos)
+{
+	struct address_space *mapping = filp->f_mapping;
+	struct inode	*inode = mapping->host;
+	struct nova_inode_info *si = NOVA_I(inode);
+	struct nova_inode_info_header *sih = &si->header;
+	struct super_block *sb = inode->i_sb;
+	struct nova_inode *pi, inode_copy;
+	struct nova_file_write_entry *entry;
+	struct nova_file_write_entry *entryc, entry_copy;
+	struct nova_file_write_entry entry_data;
+	struct nova_inode_update update;
+	ssize_t	    written = 0;
+	loff_t pos;
+	size_t count, offset, copied;
+	unsigned long start_blk, num_blocks, ent_blks = 0;
+	unsigned long total_blocks;
+	unsigned long blocknr = 0;
+	unsigned int data_bits;
+	int allocated = 0;
+	int inplace = 0;
+	bool hole_fill = false;
+	bool update_log = false;
+	void *kmem;
+	u64 blk_off;
+	size_t bytes;
+	long status = 0;
+	timing_t inplace_write_time, memcpy_time;
+	unsigned long step = 0;
+	u64 begin_tail = 0;
+	u64 epoch_id;
+	u64 file_size;
+	u32 time;
+	ssize_t ret;
+
+
+	if (len == 0)
+		return 0;
+
+
+	NOVA_START_TIMING(inplace_write_t, inplace_write_time);
+
+	sb_start_write(inode->i_sb);
+	inode_lock(inode);
+
+	if (!access_ok(VERIFY_READ, buf, len)) {
+		ret = -EFAULT;
+		goto out;
+	}
+	pos = *ppos;
+
+	if (filp->f_flags & O_APPEND)
+		pos = i_size_read(inode);
+
+	count = len;
+
+	pi = nova_get_block(sb, sih->pi_addr);
+
+	/* nova_inode tail pointer will be updated and we make sure all other
+	 * inode fields are good before checksumming the whole structure
+	 */
+	if (nova_check_inode_integrity(sb, sih->ino, sih->pi_addr,
+			sih->alter_pi_addr, &inode_copy, 0) < 0) {
+		ret = -EIO;
+		goto out;
+	}
+
+	offset = pos & (sb->s_blocksize - 1);
+	num_blocks = ((count + offset - 1) >> sb->s_blocksize_bits) + 1;
+	total_blocks = num_blocks;
+
+	/* offset in the actual block size block */
+
+	ret = file_remove_privs(filp);
+	if (ret)
+		goto out;
+
+	inode->i_ctime = inode->i_mtime = current_time(inode);
+	time = current_time(inode).tv_sec;
+
+	epoch_id = nova_get_epoch_id(sb);
+
+	nova_dbgv("%s: epoch_id %llu, inode %lu, offset %lld, count %lu\n",
+			__func__, epoch_id, inode->i_ino, pos, count);
+	update.tail = sih->log_tail;
+	update.alter_tail = sih->alter_log_tail;
+	while (num_blocks > 0) {
+		hole_fill = false;
+		offset = pos & (nova_inode_blk_size(sih) - 1);
+		start_blk = pos >> sb->s_blocksize_bits;
+
+		ent_blks = nova_check_existing_entry(sb, inode, num_blocks,
+						start_blk, &entry, &entry_copy,
+						1, epoch_id, &inplace, 1);
+
+		entryc = (metadata_csum == 0) ? entry : &entry_copy;
+
+		if (entry && inplace) {
+			/* We can do inplace write. Find contiguous blocks */
+			blocknr = get_nvmm(sb, sih, entryc, start_blk);
+			blk_off = blocknr << PAGE_SHIFT;
+			allocated = ent_blks;
+			if (data_csum || data_parity)
+				nova_set_write_entry_updating(sb, entry, 1);
+		} else {
+			/* Allocate blocks to fill hole */
+			allocated = nova_new_data_blocks(sb, sih, &blocknr,
+					 start_blk, ent_blks, ALLOC_NO_INIT,
+					 ANY_CPU, ALLOC_FROM_HEAD);
+
+			nova_dbg_verbose("%s: alloc %d blocks @ %lu\n",
+						__func__, allocated, blocknr);
+
+			if (allocated <= 0) {
+				nova_dbg("%s alloc blocks failed!, %d\n",
+							__func__, allocated);
+				ret = allocated;
+				goto out;
+			}
+
+			hole_fill = true;
+			blk_off = nova_get_block_off(sb, blocknr,
+							sih->i_blk_type);
+		}
+
+		step++;
+		bytes = sb->s_blocksize * allocated - offset;
+		if (bytes > count)
+			bytes = count;
+
+		kmem = nova_get_block(inode->i_sb, blk_off);
+
+		if (hole_fill &&
+		    (offset || ((offset + bytes) & (PAGE_SIZE - 1)) != 0)) {
+			ret =  nova_handle_head_tail_blocks(sb, inode,
+							    pos, bytes, kmem);
+			if (ret)
+				goto out;
+
+		}
+
+		/* Now copy from user buf */
+//		nova_dbg("Write: %p\n", kmem);
+		NOVA_START_TIMING(memcpy_w_nvmm_t, memcpy_time);
+		nova_memunlock_range(sb, kmem + offset, bytes);
+		copied = bytes - memcpy_to_pmem_nocache(kmem + offset,
+						buf, bytes);
+		nova_memlock_range(sb, kmem + offset, bytes);
+		NOVA_END_TIMING(memcpy_w_nvmm_t, memcpy_time);
+
+		if (data_csum > 0 || data_parity > 0) {
+			ret = nova_protect_file_data(sb, inode, pos, bytes,
+						buf, blocknr, !hole_fill);
+			if (ret)
+				goto out;
+		}
+
+		if (pos + copied > inode->i_size)
+			file_size = cpu_to_le64(pos + copied);
+		else
+			file_size = cpu_to_le64(inode->i_size);
+
+		/* Handle hole fill write */
+		if (hole_fill) {
+			nova_init_file_write_entry(sb, sih, &entry_data,
+						epoch_id, start_blk, allocated,
+						blocknr, time, file_size);
+
+			ret = nova_append_file_write_entry(sb, pi, inode,
+						&entry_data, &update);
+			if (ret) {
+				nova_dbg("%s: append inode entry failed\n",
+								__func__);
+				ret = -ENOSPC;
+				goto out;
+			}
+		} else {
+			/* Update existing entry */
+			struct nova_log_entry_info entry_info;
+
+			entry_info.type = FILE_WRITE;
+			entry_info.epoch_id = epoch_id;
+			entry_info.trans_id = sih->trans_id;
+			entry_info.time = time;
+			entry_info.file_size = file_size;
+			entry_info.inplace = 1;
+
+			nova_inplace_update_write_entry(sb, inode, entry,
+							&entry_info);
+		}
+
+		nova_dbgv("Write: %p, %lu\n", kmem, copied);
+		if (copied > 0) {
+			status = copied;
+			written += copied;
+			pos += copied;
+			buf += copied;
+			count -= copied;
+			num_blocks -= allocated;
+		}
+		if (unlikely(copied != bytes)) {
+			nova_dbg("%s ERROR!: %p, bytes %lu, copied %lu\n",
+				__func__, kmem, bytes, copied);
+			if (status >= 0)
+				status = -EFAULT;
+		}
+		if (status < 0)
+			break;
+
+		if (hole_fill) {
+			update_log = true;
+			if (begin_tail == 0)
+				begin_tail = update.curr_entry;
+		}
+	}
+
+	data_bits = blk_type_to_shift[sih->i_blk_type];
+	sih->i_blocks += (total_blocks << (data_bits - sb->s_blocksize_bits));
+
+	inode->i_blocks = sih->i_blocks;
+
+	if (update_log) {
+		nova_memunlock_inode(sb, pi);
+		nova_update_inode(sb, inode, pi, &update, 1);
+		nova_memlock_inode(sb, pi);
+		NOVA_STATS_ADD(inplace_new_blocks, 1);
+
+		/* Update file tree */
+		ret = nova_reassign_file_tree(sb, sih, begin_tail);
+		if (ret)
+			goto out;
+	}
+
+	ret = written;
+	NOVA_STATS_ADD(inplace_write_breaks, step);
+	nova_dbgv("blocks: %lu, %lu\n", inode->i_blocks, sih->i_blocks);
+
+	*ppos = pos;
+	if (pos > inode->i_size) {
+		i_size_write(inode, pos);
+		sih->i_size = pos;
+	}
+
+	sih->trans_id++;
+out:
+	if (ret < 0)
+		nova_cleanup_incomplete_write(sb, sih, blocknr, allocated,
+						begin_tail, update.tail);
+
+	inode_unlock(inode);
+	sb_end_write(inode->i_sb);
+	NOVA_END_TIMING(inplace_write_t, inplace_write_time);
+	NOVA_STATS_ADD(inplace_write_bytes, written);
+	return ret;
+}
+
+/* Check if existing entry overlap with vma regions */
+int nova_check_overlap_vmas(struct super_block *sb,
+	struct nova_inode_info_header *sih,
+	unsigned long pgoff, unsigned long num_pages)
+{
+	unsigned long start_pgoff = 0;
+	unsigned long num = 0;
+	unsigned long i;
+	struct vma_item *item;
+	struct rb_node *temp;
+	int ret = 0;
+
+	if (sih->num_vmas == 0)
+		return 0;
+
+	temp = rb_first(&sih->vma_tree);
+	while (temp) {
+		item = container_of(temp, struct vma_item, node);
+		temp = rb_next(temp);
+		ret = nova_get_vma_overlap_range(sb, sih, item->vma, pgoff,
+					num_pages, &start_pgoff, &num);
+		if (ret) {
+			for (i = 0; i < num; i++) {
+				if (nova_get_write_entry(sb, sih,
+							start_pgoff + i))
+					return 1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+ * return > 0, # of blocks mapped or allocated.
+ * return = 0, if plain lookup failed.
+ * return < 0, error case.
+ */
+int nova_dax_get_blocks(struct inode *inode, sector_t iblock,
+	unsigned long max_blocks, u32 *bno, bool *new, bool *boundary,
+	int create, bool taking_lock)
+{
+	struct super_block *sb = inode->i_sb;
+	struct nova_inode *pi;
+	struct nova_inode_info *si = NOVA_I(inode);
+	struct nova_inode_info_header *sih = &si->header;
+	struct nova_file_write_entry *entry = NULL;
+	struct nova_file_write_entry *entryc, entry_copy;
+	struct nova_file_write_entry entry_data;
+	struct nova_inode_update update;
+	u32 time;
+	unsigned int data_bits;
+	unsigned long nvmm = 0;
+	unsigned long blocknr = 0;
+	u64 epoch_id;
+	int num_blocks = 0;
+	int inplace = 0;
+	int allocated = 0;
+	int locked = 0;
+	int check_next = 1;
+	int ret = 0;
+	timing_t get_block_time;
+
+
+	if (max_blocks == 0)
+		return 0;
+
+	NOVA_START_TIMING(dax_get_block_t, get_block_time);
+
+	nova_dbgv("%s: pgoff %lu, num %lu, create %d\n",
+				__func__, iblock, max_blocks, create);
+
+	epoch_id = nova_get_epoch_id(sb);
+
+	if (taking_lock)
+		check_next = 0;
+
+again:
+	num_blocks = nova_check_existing_entry(sb, inode, max_blocks,
+					iblock, &entry, &entry_copy, check_next,
+					epoch_id, &inplace, locked);
+
+	entryc = (metadata_csum == 0) ? entry : &entry_copy;
+
+	if (entry) {
+		if (create == 0 || inplace) {
+			nvmm = get_nvmm(sb, sih, entryc, iblock);
+			nova_dbgv("%s: found pgoff %lu, block %lu\n",
+					__func__, iblock, nvmm);
+			goto out;
+		}
+	}
+
+	if (create == 0) {
+		num_blocks = 0;
+		goto out1;
+	}
+
+	if (taking_lock && locked == 0) {
+		inode_lock(inode);
+		locked = 1;
+		/* Check again incase someone has done it for us */
+		check_next = 1;
+		goto again;
+	}
+
+	pi = nova_get_inode(sb, inode);
+	inode->i_ctime = inode->i_mtime = current_time(inode);
+	time = current_time(inode).tv_sec;
+	update.tail = sih->log_tail;
+	update.alter_tail = sih->alter_log_tail;
+
+	/* Return initialized blocks to the user */
+	allocated = nova_new_data_blocks(sb, sih, &blocknr, iblock,
+				 num_blocks, ALLOC_INIT_ZERO, ANY_CPU,
+				 ALLOC_FROM_HEAD);
+	if (allocated <= 0) {
+		nova_dbgv("%s alloc blocks failed %d\n", __func__,
+							allocated);
+		ret = allocated;
+		goto out;
+	}
+
+	num_blocks = allocated;
+	/* Do not extend file size */
+	nova_init_file_write_entry(sb, sih, &entry_data,
+					epoch_id, iblock, num_blocks,
+					blocknr, time, inode->i_size);
+
+	ret = nova_append_file_write_entry(sb, pi, inode,
+				&entry_data, &update);
+	if (ret) {
+		nova_dbgv("%s: append inode entry failed\n", __func__);
+		ret = -ENOSPC;
+		goto out;
+	}
+
+	nvmm = blocknr;
+	data_bits = blk_type_to_shift[sih->i_blk_type];
+	sih->i_blocks += (num_blocks << (data_bits - sb->s_blocksize_bits));
+
+	nova_memunlock_inode(sb, pi);
+	nova_update_inode(sb, inode, pi, &update, 1);
+	nova_memlock_inode(sb, pi);
+
+	ret = nova_reassign_file_tree(sb, sih, update.curr_entry);
+	if (ret) {
+		nova_dbgv("%s: nova_reassign_file_tree failed: %d\n",
+			  __func__,  ret);
+		goto out;
+	}
+	inode->i_blocks = sih->i_blocks;
+	sih->trans_id++;
+	NOVA_STATS_ADD(dax_new_blocks, 1);
+
+//	set_buffer_new(bh);
+out:
+	if (ret < 0) {
+		nova_cleanup_incomplete_write(sb, sih, blocknr, allocated,
+						0, update.tail);
+		num_blocks = ret;
+		goto out1;
+	}
+
+	*bno = nvmm;
+//	if (num_blocks > 1)
+//		bh->b_size = sb->s_blocksize * num_blocks;
+
+out1:
+	if (taking_lock && locked)
+		inode_unlock(inode);
+
+	NOVA_END_TIMING(dax_get_block_t, get_block_time);
+	return num_blocks;
+}
+
+int nova_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
+	unsigned int flags, struct iomap *iomap, bool taking_lock)
+{
+	struct nova_sb_info *sbi = NOVA_SB(inode->i_sb);
+	unsigned int blkbits = inode->i_blkbits;
+	unsigned long first_block = offset >> blkbits;
+	unsigned long max_blocks = (length + (1 << blkbits) - 1) >> blkbits;
+	bool new = false, boundary = false;
+	u32 bno;
+	int ret;
+
+	ret = nova_dax_get_blocks(inode, first_block, max_blocks, &bno, &new,
+				  &boundary, flags & IOMAP_WRITE, taking_lock);
+	if (ret < 0) {
+		nova_dbgv("%s: nova_dax_get_blocks failed %d", __func__, ret);
+		return ret;
+	}
+
+	iomap->flags = 0;
+	iomap->bdev = inode->i_sb->s_bdev;
+	iomap->dax_dev = sbi->s_dax_dev;
+	iomap->offset = (u64)first_block << blkbits;
+
+	if (ret == 0) {
+		iomap->type = IOMAP_HOLE;
+		iomap->blkno = IOMAP_NULL_BLOCK;
+		iomap->length = 1 << blkbits;
+	} else {
+		iomap->type = IOMAP_MAPPED;
+		iomap->blkno = (sector_t)bno << (blkbits - 9);
+		iomap->length = (u64)ret << blkbits;
+		iomap->flags |= IOMAP_F_MERGED;
+	}
+
+	if (new)
+		iomap->flags |= IOMAP_F_NEW;
+	return 0;
+}
+
+int nova_iomap_end(struct inode *inode, loff_t offset, loff_t length,
+	ssize_t written, unsigned int flags, struct iomap *iomap)
+{
+	if (iomap->type == IOMAP_MAPPED &&
+			written < length &&
+			(flags & IOMAP_WRITE))
+		truncate_pagecache(inode, inode->i_size);
+	return 0;
+}
+
+
+static int nova_iomap_begin_lock(struct inode *inode, loff_t offset,
+	loff_t length, unsigned int flags, struct iomap *iomap)
+{
+	return nova_iomap_begin(inode, offset, length, flags, iomap, true);
+}
+
+static struct iomap_ops nova_iomap_ops_lock = {
+	.iomap_begin	= nova_iomap_begin_lock,
+	.iomap_end	= nova_iomap_end,
+};
+
+
+static int nova_dax_huge_fault(struct vm_fault *vmf,
+			      enum page_entry_size pe_size)
+{
+	int ret = 0;
+	timing_t fault_time;
+	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
+	struct inode *inode = mapping->host;
+
+	NOVA_START_TIMING(pmd_fault_t, fault_time);
+
+	nova_dbgv("%s: inode %lu, pgoff %lu\n",
+		  __func__, inode->i_ino, vmf->pgoff);
+
+	ret = dax_iomap_fault(vmf, pe_size, &nova_iomap_ops_lock);
+
+	NOVA_END_TIMING(pmd_fault_t, fault_time);
+	return ret;
+}
+
+static int nova_dax_fault(struct vm_fault *vmf)
+{
+	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
+	struct inode *inode = mapping->host;
+
+	nova_dbgv("%s: inode %lu, pgoff %lu\n",
+		  __func__, inode->i_ino, vmf->pgoff);
+
+	return nova_dax_huge_fault(vmf, PE_SIZE_PTE);
+}
+
+static int nova_dax_pfn_mkwrite(struct vm_fault *vmf)
+{
+	struct inode *inode = file_inode(vmf->vma->vm_file);
+	loff_t size;
+	int ret = 0;
+	timing_t fault_time;
+
+	NOVA_START_TIMING(pfn_mkwrite_t, fault_time);
+
+	inode_lock(inode);
+	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	if (vmf->pgoff >= size)
+		ret = VM_FAULT_SIGBUS;
+	else
+		ret = dax_pfn_mkwrite(vmf);
+	inode_unlock(inode);
+
+	NOVA_END_TIMING(pfn_mkwrite_t, fault_time);
+	return ret;
+}
+
+static inline int nova_rbtree_compare_vma(struct vma_item *curr,
+	struct vm_area_struct *vma)
+{
+	if (vma < curr->vma)
+		return -1;
+	if (vma > curr->vma)
+		return 1;
+
+	return 0;
+}
+
+static int nova_append_write_mmap_to_log(struct super_block *sb,
+	struct inode *inode, struct vma_item *item)
+{
+	struct vm_area_struct *vma = item->vma;
+	struct nova_inode *pi;
+	struct nova_mmap_entry data;
+	struct nova_inode_update update;
+	unsigned long num_pages;
+	u64 epoch_id;
+	int ret;
+
+	/* Only for csum and parity update */
+	if (data_csum == 0 && data_parity == 0)
+		return 0;
+
+	pi = nova_get_inode(sb, inode);
+	epoch_id = nova_get_epoch_id(sb);
+	update.tail = update.alter_tail = 0;
+
+	memset(&data, 0, sizeof(struct nova_mmap_entry));
+	data.entry_type = MMAP_WRITE;
+	data.epoch_id = epoch_id;
+	data.pgoff = cpu_to_le64(vma->vm_pgoff);
+	num_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	data.num_pages = cpu_to_le64(num_pages);
+	data.invalid = 0;
+
+	nova_dbgv("%s : Appending mmap log entry for inode %lu, pgoff %llu, %llu pages\n",
+			__func__, inode->i_ino,
+			data.pgoff, data.num_pages);
+
+	ret = nova_append_mmap_entry(sb, pi, inode, &data, &update, item);
+	if (ret) {
+		nova_dbg("%s: append write mmap entry failure\n", __func__);
+		goto out;
+	}
+
+	nova_memunlock_inode(sb, pi);
+	nova_update_inode(sb, inode, pi, &update, 1);
+	nova_memlock_inode(sb, pi);
+out:
+	return ret;
+}
+
+int nova_insert_write_vma(struct vm_area_struct *vma)
+{
+	struct address_space *mapping = vma->vm_file->f_mapping;
+	struct inode *inode = mapping->host;
+	struct nova_inode_info *si = NOVA_I(inode);
+	struct nova_inode_info_header *sih = &si->header;
+	struct super_block *sb = inode->i_sb;
+	struct nova_sb_info *sbi = NOVA_SB(sb);
+	unsigned long flags = VM_SHARED | VM_WRITE;
+	struct vma_item *item, *curr;
+	struct rb_node **temp, *parent;
+	int compVal;
+	int insert = 0;
+	int ret;
+	timing_t insert_vma_time;
+
+
+	if ((vma->vm_flags & flags) != flags)
+		return 0;
+
+	NOVA_START_TIMING(insert_vma_t, insert_vma_time);
+
+	item = nova_alloc_vma_item(sb);
+	if (!item) {
+		NOVA_END_TIMING(insert_vma_t, insert_vma_time);
+		return -ENOMEM;
+	}
+
+	item->vma = vma;
+
+	nova_dbgv("Inode %lu insert vma %p, start 0x%lx, end 0x%lx, pgoff %lu\n",
+			inode->i_ino, vma, vma->vm_start, vma->vm_end,
+			vma->vm_pgoff);
+
+	inode_lock(inode);
+
+	/* Append to log */
+	ret = nova_append_write_mmap_to_log(sb, inode, item);
+	if (ret)
+		goto out;
+
+	temp = &(sih->vma_tree.rb_node);
+	parent = NULL;
+
+	while (*temp) {
+		curr = container_of(*temp, struct vma_item, node);
+		compVal = nova_rbtree_compare_vma(curr, vma);
+		parent = *temp;
+
+		if (compVal == -1) {
+			temp = &((*temp)->rb_left);
+		} else if (compVal == 1) {
+			temp = &((*temp)->rb_right);
+		} else {
+			nova_dbg("%s: vma %p already exists\n",
+				__func__, vma);
+			kfree(item);
+			goto out;
+		}
+	}
+
+	rb_link_node(&item->node, parent, temp);
+	rb_insert_color(&item->node, &sih->vma_tree);
+
+	sih->num_vmas++;
+	if (sih->num_vmas == 1)
+		insert = 1;
+
+	sih->trans_id++;
+out:
+	inode_unlock(inode);
+
+	if (insert) {
+		mutex_lock(&sbi->vma_mutex);
+		list_add_tail(&sih->list, &sbi->mmap_sih_list);
+		mutex_unlock(&sbi->vma_mutex);
+	}
+
+	NOVA_END_TIMING(insert_vma_t, insert_vma_time);
+	return ret;
+}
+
+static int nova_remove_write_vma(struct vm_area_struct *vma)
+{
+	struct address_space *mapping = vma->vm_file->f_mapping;
+	struct inode *inode = mapping->host;
+	struct nova_inode_info *si = NOVA_I(inode);
+	struct nova_inode_info_header *sih = &si->header;
+	struct super_block *sb = inode->i_sb;
+	struct nova_sb_info *sbi = NOVA_SB(sb);
+	struct vma_item *curr = NULL;
+	struct rb_node *temp;
+	int compVal;
+	int found = 0;
+	int remove = 0;
+	timing_t remove_vma_time;
+
+
+	NOVA_START_TIMING(remove_vma_t, remove_vma_time);
+	inode_lock(inode);
+
+	temp = sih->vma_tree.rb_node;
+	while (temp) {
+		curr = container_of(temp, struct vma_item, node);
+		compVal = nova_rbtree_compare_vma(curr, vma);
+
+		if (compVal == -1) {
+			temp = temp->rb_left;
+		} else if (compVal == 1) {
+			temp = temp->rb_right;
+		} else {
+			nova_reset_vma_csum_parity(sb, curr);
+			rb_erase(&curr->node, &sih->vma_tree);
+			found = 1;
+			break;
+		}
+	}
+
+	if (found) {
+		sih->num_vmas--;
+		if (sih->num_vmas == 0)
+			remove = 1;
+	}
+
+	inode_unlock(inode);
+
+	if (found) {
+		nova_dbgv("Inode %lu remove vma %p, start 0x%lx, end 0x%lx, pgoff %lu\n",
+			  inode->i_ino,	curr->vma, curr->vma->vm_start,
+			  curr->vma->vm_end, curr->vma->vm_pgoff);
+		nova_free_vma_item(sb, curr);
+	}
+
+	if (remove) {
+		mutex_lock(&sbi->vma_mutex);
+		list_del(&sih->list);
+		mutex_unlock(&sbi->vma_mutex);
+	}
+
+	NOVA_END_TIMING(remove_vma_t, remove_vma_time);
+	return 0;
+}
+
+static int nova_restore_page_write(struct vm_area_struct *vma,
+	unsigned long address)
+{
+	struct mm_struct *mm = vma->vm_mm;
+
+
+	down_write(&mm->mmap_sem);
+
+	nova_dbgv("Restore vma %p write, start 0x%lx, end 0x%lx, address 0x%lx\n",
+		  vma, vma->vm_start, vma->vm_end, address);
+
+	/* Restore single page write */
+	nova_mmap_to_new_blocks(vma, address);
+
+	up_write(&mm->mmap_sem);
+
+	return 0;
+}
+
+static void nova_vma_open(struct vm_area_struct *vma)
+{
+	struct address_space *mapping = vma->vm_file->f_mapping;
+	struct inode *inode = mapping->host;
+
+	nova_dbg_mmap4k("[%s:%d] inode %lu, MMAP 4KPAGE vm_start(0x%lx), vm_end(0x%lx), vm pgoff %lu, %lu blocks, vm_flags(0x%lx), vm_page_prot(0x%lx)\n",
+			__func__, __LINE__,
+			inode->i_ino, vma->vm_start, vma->vm_end,
+			vma->vm_pgoff,
+			(vma->vm_end - vma->vm_start) >> PAGE_SHIFT,
+			vma->vm_flags,
+			pgprot_val(vma->vm_page_prot));
+
+	nova_insert_write_vma(vma);
+}
+
+static void nova_vma_close(struct vm_area_struct *vma)
+{
+	nova_dbgv("[%s:%d] MMAP 4KPAGE vm_start(0x%lx), vm_end(0x%lx), vm_flags(0x%lx), vm_page_prot(0x%lx)\n",
+		  __func__, __LINE__, vma->vm_start, vma->vm_end,
+		  vma->vm_flags, pgprot_val(vma->vm_page_prot));
+
+	vma->original_write = 0;
+	nova_remove_write_vma(vma);
+}
+
+const struct vm_operations_struct nova_dax_vm_ops = {
+	.fault	= nova_dax_fault,
+	.huge_fault = nova_dax_huge_fault,
+	.page_mkwrite = nova_dax_fault,
+	.pfn_mkwrite = nova_dax_pfn_mkwrite,
+	.open = nova_vma_open,
+	.close = nova_vma_close,
+	.dax_cow = nova_restore_page_write,
+};
+