Merge branch 'pt/fsmonitor-linux' into seen

The fsmonitor daemon has been implemented for Linux.

Comments?

* pt/fsmonitor-linux:
  fsmonitor: implement filesystem change listener for Linux

Documentation/config/fsmonitor--daemon.adoc

@@ -4,8 +4,8 @@ fsmonitor.allowRemote::
     behavior.  Only respected when `core.fsmonitor` is set to `true`.
 
 fsmonitor.socketDir::
-    This Mac OS-specific option, if set, specifies the directory in
+    This Mac OS and Linux-specific option, if set, specifies the directory in
     which to create the Unix domain socket used for communication
     between the fsmonitor daemon and various Git commands. The directory must
-    reside on a native Mac OS filesystem.  Only respected when `core.fsmonitor`
+    reside on a native filesystem.  Only respected when `core.fsmonitor`
     is set to `true`.

Documentation/git-fsmonitor--daemon.adoc

@@ -76,9 +76,9 @@ repositories; this may be overridden by setting `fsmonitor.allowRemote` to
 correctly with all network-mounted repositories, so such use is considered
 experimental.
 
-On Mac OS, the inter-process communication (IPC) between various Git
+On Mac OS and Linux, the inter-process communication (IPC) between various Git
 commands and the fsmonitor daemon is done via a Unix domain socket (UDS) -- a
-special type of file -- which is supported by native Mac OS filesystems,
+special type of file -- which is supported by native Mac OS and Linux filesystems,
 but not on network-mounted filesystems, NTFS, or FAT32.  Other filesystems
 may or may not have the needed support; the fsmonitor daemon is not guaranteed
 to work with these filesystems and such use is considered experimental.
@@ -87,13 +87,33 @@ By default, the socket is created in the `.git` directory.  However, if the
 `.git` directory is on a network-mounted filesystem, it will instead be
 created at `$HOME/.git-fsmonitor-*` unless `$HOME` itself is on a
 network-mounted filesystem, in which case you must set the configuration
-variable `fsmonitor.socketDir` to the path of a directory on a Mac OS native
+variable `fsmonitor.socketDir` to the path of a directory on a native
 filesystem in which to create the socket file.
 
 If none of the above directories (`.git`, `$HOME`, or `fsmonitor.socketDir`)
-is on a native Mac OS file filesystem the fsmonitor daemon will report an
+is on a native filesystem the fsmonitor daemon will report an
 error that will cause the daemon and the currently running command to exit.
 
+LINUX CAVEATS
+~~~~~~~~~~~~~
+
+On Linux, the fsmonitor daemon uses inotify to monitor filesystem events.
+The inotify system has per-user limits on the number of watches that can
+be created.  The default limit is typically 8192 watches per user.
+
+For large repositories with many directories, you may need to increase
+this limit.  Check the current limit with:
+
+    cat /proc/sys/fs/inotify/max_user_watches
+
+To temporarily increase the limit:
+
+    sudo sysctl fs.inotify.max_user_watches=65536
+
+To make the change permanent, add to `/etc/sysctl.conf`:
+
+    fs.inotify.max_user_watches=65536
+
 CONFIGURATION
 -------------
 

builtin/fsmonitor--daemon.c

@@ -671,7 +671,7 @@ static int do_handle_client(struct fsmonitor_daemon_state *state,
 	const struct fsmonitor_batch *batch;
 	struct fsmonitor_batch *remainder = NULL;
 	intmax_t count = 0, duplicates = 0;
-	kh_str_t *shown;
+	kh_str_t *shown = NULL;
 	int hash_ret;
 	int do_trivial = 0;
 	int do_flush = 0;
@@ -909,8 +909,6 @@ static int do_handle_client(struct fsmonitor_daemon_state *state,
 		total_response_len += payload.len;
 	}
 
-	kh_release_str(shown);
-
 	pthread_mutex_lock(&state->main_lock);
 
 	if (token_data->client_ref_count > 0)
@@ -954,6 +952,7 @@ static int do_handle_client(struct fsmonitor_daemon_state *state,
 	trace2_data_intmax("fsmonitor", the_repository, "response/count/duplicates", duplicates);
 
 cleanup:
+	kh_destroy_str(shown);
 	strbuf_release(&response_token);
 	strbuf_release(&requested_token_id);
 	strbuf_release(&payload);
@@ -1405,6 +1404,7 @@ static int fsmonitor_run_daemon(void)
 done:
 	pthread_cond_destroy(&state.cookies_cond);
 	pthread_mutex_destroy(&state.main_lock);
+	hashmap_clear(&state.cookies);
 	fsm_listen__dtor(&state);
 	fsm_health__dtor(&state);
 

compat/fsmonitor/fsm-health-linux.c

@@ -0,0 +1,33 @@
+#include "git-compat-util.h"
+#include "config.h"
+#include "fsmonitor-ll.h"
+#include "fsm-health.h"
+#include "fsmonitor--daemon.h"
+
+/*
+ * The Linux fsmonitor implementation uses inotify which has its own
+ * mechanisms for detecting filesystem unmount and other events that
+ * would require the daemon to shutdown.  Therefore, we don't need
+ * a separate health thread like Windows does.
+ *
+ * These stub functions satisfy the interface requirements.
+ */
+
+int fsm_health__ctor(struct fsmonitor_daemon_state *state UNUSED)
+{
+	return 0;
+}
+
+void fsm_health__dtor(struct fsmonitor_daemon_state *state UNUSED)
+{
+	return;
+}
+
+void fsm_health__loop(struct fsmonitor_daemon_state *state UNUSED)
+{
+	return;
+}
+
+void fsm_health__stop_async(struct fsmonitor_daemon_state *state UNUSED)
+{
+}

compat/fsmonitor/fsm-ipc-linux.c

@@ -0,0 +1,61 @@
+#define USE_THE_REPOSITORY_VARIABLE
+
+#include "git-compat-util.h"
+#include "config.h"
+#include "gettext.h"
+#include "hex.h"
+#include "path.h"
+#include "repository.h"
+#include "strbuf.h"
+#include "fsmonitor-ll.h"
+#include "fsmonitor-ipc.h"
+#include "fsmonitor-path-utils.h"
+
+static GIT_PATH_FUNC(fsmonitor_ipc__get_default_path, "fsmonitor--daemon.ipc")
+
+const char *fsmonitor_ipc__get_path(struct repository *r)
+{
+	static const char *ipc_path = NULL;
+	git_SHA_CTX sha1ctx;
+	char *sock_dir = NULL;
+	struct strbuf ipc_file = STRBUF_INIT;
+	unsigned char hash[GIT_SHA1_RAWSZ];
+
+	if (!r)
+		BUG("No repository passed into fsmonitor_ipc__get_path");
+
+	if (ipc_path)
+		return ipc_path;
+
+	/* By default the socket file is created in the .git directory */
+	if (fsmonitor__is_fs_remote(r->gitdir) < 1) {
+		ipc_path = fsmonitor_ipc__get_default_path();
+		return ipc_path;
+	}
+
+	if (!r->worktree)
+		BUG("repository has no worktree");
+
+	git_SHA1_Init(&sha1ctx);
+	git_SHA1_Update(&sha1ctx, r->worktree, strlen(r->worktree));
+	git_SHA1_Final(hash, &sha1ctx);
+
+	repo_config_get_string(r, "fsmonitor.socketdir", &sock_dir);
+
+	/* Create the socket file in either socketDir or $HOME */
+	if (sock_dir && *sock_dir) {
+		strbuf_addf(&ipc_file, "%s/.git-fsmonitor-%s",
+			    sock_dir, hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1]));
+	} else {
+		strbuf_addf(&ipc_file, "~/.git-fsmonitor-%s",
+			    hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1]));
+	}
+	free(sock_dir);
+
+	ipc_path = interpolate_path(ipc_file.buf, 1);
+	if (!ipc_path)
+		die(_("Invalid path: %s"), ipc_file.buf);
+
+	strbuf_release(&ipc_file);
+	return ipc_path;
+}

compat/fsmonitor/fsm-listen-linux.c

@@ -0,0 +1,740 @@
+#include "git-compat-util.h"
+#include "dir.h"
+#include "fsmonitor-ll.h"
+#include "fsm-listen.h"
+#include "fsmonitor--daemon.h"
+#include "fsmonitor-path-utils.h"
+#include "gettext.h"
+#include "simple-ipc.h"
+#include "string-list.h"
+#include "trace.h"
+
+#include <dirent.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <sys/inotify.h>
+#include <sys/stat.h>
+
+/*
+ * Safe value to bitwise OR with rest of mask for
+ * kernels that do not support IN_MASK_CREATE
+ */
+#ifndef IN_MASK_CREATE
+#define IN_MASK_CREATE 0x00000000
+#endif
+
+enum shutdown_reason {
+	SHUTDOWN_CONTINUE = 0,
+	SHUTDOWN_STOP,
+	SHUTDOWN_ERROR,
+	SHUTDOWN_FORCE
+};
+
+struct watch_entry {
+	struct hashmap_entry ent;
+	int wd;
+	uint32_t cookie;
+	const char *dir;
+};
+
+struct rename_entry {
+	struct hashmap_entry ent;
+	time_t whence;
+	uint32_t cookie;
+	const char *dir;
+};
+
+struct fsm_listen_data {
+	int fd_inotify;
+	enum shutdown_reason shutdown;
+	struct hashmap watches;
+	struct hashmap renames;
+	struct hashmap revwatches;
+};
+
+static int watch_entry_cmp(const void *cmp_data UNUSED,
+			   const struct hashmap_entry *eptr,
+			   const struct hashmap_entry *entry_or_key,
+			   const void *keydata UNUSED)
+{
+	const struct watch_entry *e1, *e2;
+
+	e1 = container_of(eptr, const struct watch_entry, ent);
+	e2 = container_of(entry_or_key, const struct watch_entry, ent);
+	return e1->wd != e2->wd;
+}
+
+static int revwatches_entry_cmp(const void *cmp_data UNUSED,
+				const struct hashmap_entry *eptr,
+				const struct hashmap_entry *entry_or_key,
+				const void *keydata UNUSED)
+{
+	const struct watch_entry *e1, *e2;
+
+	e1 = container_of(eptr, const struct watch_entry, ent);
+	e2 = container_of(entry_or_key, const struct watch_entry, ent);
+	return strcmp(e1->dir, e2->dir);
+}
+
+static int rename_entry_cmp(const void *cmp_data UNUSED,
+			    const struct hashmap_entry *eptr,
+			    const struct hashmap_entry *entry_or_key,
+			    const void *keydata UNUSED)
+{
+	const struct rename_entry *e1, *e2;
+
+	e1 = container_of(eptr, const struct rename_entry, ent);
+	e2 = container_of(entry_or_key, const struct rename_entry, ent);
+	return e1->cookie != e2->cookie;
+}
+
+/*
+ * Register an inotify watch, add watch descriptor to path mapping
+ * and the reverse mapping.
+ */
+static int add_watch(const char *path, struct fsm_listen_data *data)
+{
+	const char *interned = strintern(path);
+	struct watch_entry *w1, *w2;
+
+	/* add the inotify watch, don't allow watches to be modified */
+	int wd = inotify_add_watch(data->fd_inotify, interned,
+				   (IN_ALL_EVENTS | IN_ONLYDIR | IN_MASK_CREATE)
+				    ^ IN_ACCESS ^ IN_CLOSE ^ IN_OPEN);
+	if (wd < 0) {
+		if (errno == ENOENT || errno == ENOTDIR)
+			return 0; /* directory was deleted or is not a directory */
+		if (errno == EEXIST)
+			return 0; /* watch already exists, no action needed */
+		return error_errno(_("inotify_add_watch('%s') failed"), interned);
+	}
+
+	/* add watch descriptor -> directory mapping */
+	CALLOC_ARRAY(w1, 1);
+	w1->wd = wd;
+	w1->dir = interned;
+	hashmap_entry_init(&w1->ent, memhash(&w1->wd, sizeof(int)));
+	hashmap_add(&data->watches, &w1->ent);
+
+	/* add directory -> watch descriptor mapping */
+	CALLOC_ARRAY(w2, 1);
+	w2->wd = wd;
+	w2->dir = interned;
+	hashmap_entry_init(&w2->ent, strhash(w2->dir));
+	hashmap_add(&data->revwatches, &w2->ent);
+
+	return 0;
+}
+
+/*
+ * Remove the inotify watch, the watch descriptor to path mapping
+ * and the reverse mapping.
+ */
+static void remove_watch(struct watch_entry *w, struct fsm_listen_data *data)
+{
+	struct watch_entry k1, k2, *w1, *w2;
+
+	/* remove watch, ignore error if kernel already did it */
+	if (inotify_rm_watch(data->fd_inotify, w->wd) && errno != EINVAL)
+		error_errno(_("inotify_rm_watch() failed"));
+
+	k1.wd = w->wd;
+	hashmap_entry_init(&k1.ent, memhash(&k1.wd, sizeof(int)));
+	w1 = hashmap_remove_entry(&data->watches, &k1, ent, NULL);
+	if (!w1)
+		BUG("Double remove of watch for '%s'", w->dir);
+
+	if (w1->cookie)
+		BUG("Removing watch for '%s' which has a pending rename", w1->dir);
+
+	k2.dir = w->dir;
+	hashmap_entry_init(&k2.ent, strhash(k2.dir));
+	w2 = hashmap_remove_entry(&data->revwatches, &k2, ent, NULL);
+	if (!w2)
+		BUG("Double remove of reverse watch for '%s'", w->dir);
+
+	/* w1->dir and w2->dir are interned strings, we don't own them */
+	free(w1);
+	free(w2);
+}
+
+/*
+ * Check for stale directory renames.
+ *
+ * https://man7.org/linux/man-pages/man7/inotify.7.html
+ *
+ * Allow for some small timeout to account for the fact that insertion of the
+ * IN_MOVED_FROM+IN_MOVED_TO event pair is not atomic, and the possibility that
+ * there may not be any IN_MOVED_TO event.
+ *
+ * If the IN_MOVED_TO event is not received within the timeout then events have
+ * been missed and the monitor is in an inconsistent state with respect to the
+ * filesystem.
+ */
+static int check_stale_dir_renames(struct hashmap *renames, time_t max_age)
+{
+	struct rename_entry *re;
+	struct hashmap_iter iter;
+
+	hashmap_for_each_entry(renames, &iter, re, ent) {
+		if (re->whence <= max_age)
+			return -1;
+	}
+	return 0;
+}
+
+/*
+ * Track pending renames.
+ *
+ * Tracking is done via an event cookie to watch descriptor mapping.
+ *
+ * A rename is not complete until matching an IN_MOVED_TO event is received
+ * for a corresponding IN_MOVED_FROM event.
+ */
+static void add_dir_rename(uint32_t cookie, const char *path,
+			   struct fsm_listen_data *data)
+{
+	struct watch_entry k, *w;
+	struct rename_entry *re;
+
+	/* lookup the watch descriptor for the given path */
+	k.dir = path;
+	hashmap_entry_init(&k.ent, strhash(path));
+	w = hashmap_get_entry(&data->revwatches, &k, ent, NULL);
+	if (!w) {
+		/*
+		 * This can happen in rare cases where the directory was
+		 * moved before we had a chance to add a watch on it.
+		 * Just ignore this rename.
+		 */
+		trace_printf_key(&trace_fsmonitor,
+				 "no watch found for rename from '%s'", path);
+		return;
+	}
+	w->cookie = cookie;
+
+	/* add the pending rename to match against later */
+	CALLOC_ARRAY(re, 1);
+	re->dir = w->dir;
+	re->cookie = w->cookie;
+	re->whence = time(NULL);
+	hashmap_entry_init(&re->ent, memhash(&re->cookie, sizeof(uint32_t)));
+	hashmap_add(&data->renames, &re->ent);
+}
+
+/*
+ * Handle directory renames
+ *
+ * Once an IN_MOVED_TO event is received, lookup the rename tracking information
+ * via the event cookie and use this information to update the watch.
+ */
+static void rename_dir(uint32_t cookie, const char *path,
+		       struct fsm_listen_data *data)
+{
+	struct rename_entry rek, *re;
+	struct watch_entry k, *w;
+
+	/* lookup a pending rename to match */
+	rek.cookie = cookie;
+	hashmap_entry_init(&rek.ent, memhash(&rek.cookie, sizeof(uint32_t)));
+	re = hashmap_get_entry(&data->renames, &rek, ent, NULL);
+	if (re) {
+		k.dir = re->dir;
+		hashmap_entry_init(&k.ent, strhash(k.dir));
+		w = hashmap_get_entry(&data->revwatches, &k, ent, NULL);
+		if (w) {
+			w->cookie = 0; /* rename handled */
+			remove_watch(w, data);
+			if (add_watch(path, data))
+				trace_printf_key(&trace_fsmonitor,
+						 "failed to add watch for renamed dir '%s'",
+						 path);
+		} else {
+			/* Directory was moved out of watch tree */
+			trace_printf_key(&trace_fsmonitor,
+					 "No matching watch for rename to '%s'", path);
+		}
+		hashmap_remove_entry(&data->renames, &rek, ent, NULL);
+		free(re);
+	} else {
+		/* Directory was moved from outside the watch tree */
+		trace_printf_key(&trace_fsmonitor,
+				 "No matching cookie for rename to '%s'", path);
+	}
+}
+
+/*
+ * Recursively add watches to every directory under path
+ */
+static int register_inotify(const char *path,
+			    struct fsmonitor_daemon_state *state,
+			    struct fsmonitor_batch *batch)
+{
+	DIR *dir;
+	const char *rel;
+	struct strbuf current = STRBUF_INIT;
+	struct dirent *de;
+	struct stat fs;
+	int ret = -1;
+
+	dir = opendir(path);
+	if (!dir) {
+		if (errno == ENOENT || errno == ENOTDIR)
+			return 0; /* directory was deleted */
+		return error_errno(_("opendir('%s') failed"), path);
+	}
+
+	while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) {
+		strbuf_reset(&current);
+		strbuf_addf(&current, "%s/%s", path, de->d_name);
+		if (lstat(current.buf, &fs)) {
+			if (errno == ENOENT)
+				continue; /* file was deleted */
+			error_errno(_("lstat('%s') failed"), current.buf);
+			goto failed;
+		}
+
+		/* recurse into directory */
+		if (S_ISDIR(fs.st_mode)) {
+			if (add_watch(current.buf, state->listen_data))
+				goto failed;
+			if (register_inotify(current.buf, state, batch))
+				goto failed;
+		} else if (batch) {
+			rel = current.buf + state->path_worktree_watch.len + 1;
+			trace_printf_key(&trace_fsmonitor, "explicitly adding '%s'", rel);
+			fsmonitor_batch__add_path(batch, rel);
+		}
+	}
+	ret = 0;
+
+failed:
+	strbuf_release(&current);
+	if (closedir(dir) < 0)
+		return error_errno(_("closedir('%s') failed"), path);
+	return ret;
+}
+
+static int em_rename_dir_from(uint32_t mask)
+{
+	return ((mask & IN_ISDIR) && (mask & IN_MOVED_FROM));
+}
+
+static int em_rename_dir_to(uint32_t mask)
+{
+	return ((mask & IN_ISDIR) && (mask & IN_MOVED_TO));
+}
+
+static int em_remove_watch(uint32_t mask)
+{
+	return (mask & IN_DELETE_SELF);
+}
+
+static int em_dir_renamed(uint32_t mask)
+{
+	return ((mask & IN_ISDIR) && (mask & IN_MOVE));
+}
+
+static int em_dir_created(uint32_t mask)
+{
+	return ((mask & IN_ISDIR) && (mask & IN_CREATE));
+}
+
+static int em_dir_deleted(uint32_t mask)
+{
+	return ((mask & IN_ISDIR) && (mask & IN_DELETE));
+}
+
+static int em_force_shutdown(uint32_t mask)
+{
+	return (mask & IN_UNMOUNT) || (mask & IN_Q_OVERFLOW);
+}
+
+static int em_ignore(uint32_t mask)
+{
+	return (mask & IN_IGNORED) || (mask & IN_MOVE_SELF);
+}
+
+static void log_mask_set(const char *path, uint32_t mask)
+{
+	struct strbuf msg = STRBUF_INIT;
+
+	if (mask & IN_ACCESS)
+		strbuf_addstr(&msg, "IN_ACCESS|");
+	if (mask & IN_MODIFY)
+		strbuf_addstr(&msg, "IN_MODIFY|");
+	if (mask & IN_ATTRIB)
+		strbuf_addstr(&msg, "IN_ATTRIB|");
+	if (mask & IN_CLOSE_WRITE)
+		strbuf_addstr(&msg, "IN_CLOSE_WRITE|");
+	if (mask & IN_CLOSE_NOWRITE)
+		strbuf_addstr(&msg, "IN_CLOSE_NOWRITE|");
+	if (mask & IN_OPEN)
+		strbuf_addstr(&msg, "IN_OPEN|");
+	if (mask & IN_MOVED_FROM)
+		strbuf_addstr(&msg, "IN_MOVED_FROM|");
+	if (mask & IN_MOVED_TO)
+		strbuf_addstr(&msg, "IN_MOVED_TO|");
+	if (mask & IN_CREATE)
+		strbuf_addstr(&msg, "IN_CREATE|");
+	if (mask & IN_DELETE)
+		strbuf_addstr(&msg, "IN_DELETE|");
+	if (mask & IN_DELETE_SELF)
+		strbuf_addstr(&msg, "IN_DELETE_SELF|");
+	if (mask & IN_MOVE_SELF)
+		strbuf_addstr(&msg, "IN_MOVE_SELF|");
+	if (mask & IN_UNMOUNT)
+		strbuf_addstr(&msg, "IN_UNMOUNT|");
+	if (mask & IN_Q_OVERFLOW)
+		strbuf_addstr(&msg, "IN_Q_OVERFLOW|");
+	if (mask & IN_IGNORED)
+		strbuf_addstr(&msg, "IN_IGNORED|");
+	if (mask & IN_ISDIR)
+		strbuf_addstr(&msg, "IN_ISDIR|");
+
+	strbuf_strip_suffix(&msg, "|");
+
+	trace_printf_key(&trace_fsmonitor, "inotify_event: '%s', mask=%#8.8x %s",
+			 path, mask, msg.buf);
+
+	strbuf_release(&msg);
+}
+
+int fsm_listen__ctor(struct fsmonitor_daemon_state *state)
+{
+	int fd;
+	int ret = 0;
+	struct fsm_listen_data *data;
+
+	CALLOC_ARRAY(data, 1);
+	state->listen_data = data;
+	state->listen_error_code = -1;
+	data->shutdown = SHUTDOWN_ERROR;
+
+	fd = inotify_init1(O_NONBLOCK);
+	if (fd < 0) {
+		FREE_AND_NULL(state->listen_data);
+		return error_errno(_("inotify_init1() failed"));
+	}
+
+	data->fd_inotify = fd;
+
+	hashmap_init(&data->watches, watch_entry_cmp, NULL, 0);
+	hashmap_init(&data->renames, rename_entry_cmp, NULL, 0);
+	hashmap_init(&data->revwatches, revwatches_entry_cmp, NULL, 0);
+
+	if (add_watch(state->path_worktree_watch.buf, data))
+		ret = -1;
+	else if (register_inotify(state->path_worktree_watch.buf, state, NULL))
+		ret = -1;
+	else if (state->nr_paths_watching > 1) {
+		if (add_watch(state->path_gitdir_watch.buf, data))
+			ret = -1;
+		else if (register_inotify(state->path_gitdir_watch.buf, state, NULL))
+			ret = -1;
+	}
+
+	if (!ret) {
+		state->listen_error_code = 0;
+		data->shutdown = SHUTDOWN_CONTINUE;
+	}
+
+	return ret;
+}
+
+void fsm_listen__dtor(struct fsmonitor_daemon_state *state)
+{
+	struct fsm_listen_data *data;
+	struct hashmap_iter iter;
+	struct watch_entry *w;
+	struct watch_entry **to_remove;
+	size_t nr_to_remove = 0, alloc_to_remove = 0;
+	size_t i;
+	int fd;
+
+	if (!state || !state->listen_data)
+		return;
+
+	data = state->listen_data;
+	fd = data->fd_inotify;
+
+	/*
+	 * Collect all entries first, then remove them.
+	 * We can't modify the hashmap while iterating over it.
+	 */
+	to_remove = NULL;
+	hashmap_for_each_entry(&data->watches, &iter, w, ent) {
+		ALLOC_GROW(to_remove, nr_to_remove + 1, alloc_to_remove);
+		to_remove[nr_to_remove++] = w;
+	}
+
+	for (i = 0; i < nr_to_remove; i++) {
+		to_remove[i]->cookie = 0; /* ignore any pending renames */
+		remove_watch(to_remove[i], data);
+	}
+	free(to_remove);
+
+	hashmap_clear(&data->watches);
+
+	hashmap_clear(&data->revwatches); /* remove_watch freed the entries */
+
+	hashmap_clear_and_free(&data->renames, struct rename_entry, ent);
+
+	FREE_AND_NULL(state->listen_data);
+
+	if (fd && (close(fd) < 0))
+		error_errno(_("closing inotify file descriptor failed"));
+}
+
+void fsm_listen__stop_async(struct fsmonitor_daemon_state *state)
+{
+	if (state && state->listen_data &&
+	    state->listen_data->shutdown == SHUTDOWN_CONTINUE)
+		state->listen_data->shutdown = SHUTDOWN_STOP;
+}
+
+/*
+ * Process a single inotify event and queue for publication.
+ */
+static int process_event(const char *path,
+			 const struct inotify_event *event,
+			 struct fsmonitor_batch **batch,
+			 struct string_list *cookie_list,
+			 struct fsmonitor_daemon_state *state)
+{
+	const char *rel;
+	const char *last_sep;
+
+	switch (fsmonitor_classify_path_absolute(state, path)) {
+	case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX:
+	case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX:
+		/* Use just the filename of the cookie file. */
+		last_sep = find_last_dir_sep(path);
+		string_list_append(cookie_list,
+				   last_sep ? last_sep + 1 : path);
+		break;
+	case IS_INSIDE_DOT_GIT:
+	case IS_INSIDE_GITDIR:
+		break;
+	case IS_DOT_GIT:
+	case IS_GITDIR:
+		/*
+		 * If .git directory is deleted or renamed away,
+		 * we have to quit.
+		 */
+		if (em_dir_deleted(event->mask)) {
+			trace_printf_key(&trace_fsmonitor,
+					 "event: gitdir removed");
+			state->listen_data->shutdown = SHUTDOWN_FORCE;
+			goto done;
+		}
+
+		if (em_dir_renamed(event->mask)) {
+			trace_printf_key(&trace_fsmonitor,
+					 "event: gitdir renamed");
+			state->listen_data->shutdown = SHUTDOWN_FORCE;
+			goto done;
+		}
+		break;
+	case IS_WORKDIR_PATH:
+		/* normal events in the working directory */
+		if (trace_pass_fl(&trace_fsmonitor))
+			log_mask_set(path, event->mask);
+
+		if (!*batch)
+			*batch = fsmonitor_batch__new();
+
+		rel = path + state->path_worktree_watch.len + 1;
+		fsmonitor_batch__add_path(*batch, rel);
+
+		if (em_dir_deleted(event->mask))
+			break;
+
+		/* received IN_MOVE_FROM, add tracking for expected IN_MOVE_TO */
+		if (em_rename_dir_from(event->mask))
+			add_dir_rename(event->cookie, path, state->listen_data);
+
+		/* received IN_MOVE_TO, update watch to reflect new path */
+		if (em_rename_dir_to(event->mask)) {
+			rename_dir(event->cookie, path, state->listen_data);
+			if (register_inotify(path, state, *batch)) {
+				state->listen_data->shutdown = SHUTDOWN_ERROR;
+				goto done;
+			}
+		}
+
+		if (em_dir_created(event->mask)) {
+			if (add_watch(path, state->listen_data)) {
+				state->listen_data->shutdown = SHUTDOWN_ERROR;
+				goto done;
+			}
+			if (register_inotify(path, state, *batch)) {
+				state->listen_data->shutdown = SHUTDOWN_ERROR;
+				goto done;
+			}
+		}
+		break;
+	case IS_OUTSIDE_CONE:
+	default:
+		trace_printf_key(&trace_fsmonitor,
+				 "ignoring '%s'", path);
+		break;
+	}
+	return 0;
+done:
+	return -1;
+}
+
+/*
+ * Read the inotify event stream and pre-process events before further
+ * processing and eventual publishing.
+ */
+static void handle_events(struct fsmonitor_daemon_state *state)
+{
+	/* See https://man7.org/linux/man-pages/man7/inotify.7.html */
+	char buf[4096]
+		__attribute__ ((aligned(__alignof__(struct inotify_event))));
+
+	struct hashmap *watches = &state->listen_data->watches;
+	struct fsmonitor_batch *batch = NULL;
+	struct string_list cookie_list = STRING_LIST_INIT_DUP;
+	struct watch_entry k, *w;
+	struct strbuf path = STRBUF_INIT;
+	const struct inotify_event *event;
+	int fd = state->listen_data->fd_inotify;
+	ssize_t len;
+	char *ptr, *p;
+
+	for (;;) {
+		len = read(fd, buf, sizeof(buf));
+		if (len == -1) {
+			if (errno == EAGAIN || errno == EINTR)
+				goto done;
+			error_errno(_("reading inotify message stream failed"));
+			state->listen_data->shutdown = SHUTDOWN_ERROR;
+			goto done;
+		}
+
+		/* nothing to read */
+		if (len == 0)
+			goto done;
+
+		/* Loop over all events in the buffer. */
+		for (ptr = buf; ptr < buf + len;
+		     ptr += sizeof(struct inotify_event) + event->len) {
+
+			event = (const struct inotify_event *)ptr;
+
+			if (em_ignore(event->mask))
+				continue;
+
+			/* File system was unmounted or event queue overflowed */
+			if (em_force_shutdown(event->mask)) {
+				if (trace_pass_fl(&trace_fsmonitor))
+					log_mask_set("Forcing shutdown", event->mask);
+				state->listen_data->shutdown = SHUTDOWN_FORCE;
+				goto done;
+			}
+
+			k.wd = event->wd;
+			hashmap_entry_init(&k.ent, memhash(&k.wd, sizeof(int)));
+
+			w = hashmap_get_entry(watches, &k, ent, NULL);
+			if (!w) {
+				/* Watch was removed, skip event */
+				continue;
+			}
+
+			/* directory watch was removed */
+			if (em_remove_watch(event->mask)) {
+				remove_watch(w, state->listen_data);
+				continue;
+			}
+
+			strbuf_reset(&path);
+			strbuf_addf(&path, "%s/%s", w->dir, event->name);
+
+			p = fsmonitor__resolve_alias(path.buf, &state->alias);
+			if (!p)
+				p = strbuf_detach(&path, NULL);
+
+			if (process_event(p, event, &batch, &cookie_list, state)) {
+				free(p);
+				goto done;
+			}
+			free(p);
+		}
+		strbuf_reset(&path);
+		fsmonitor_publish(state, batch, &cookie_list);
+		string_list_clear(&cookie_list, 0);
+		batch = NULL;
+	}
+done:
+	strbuf_release(&path);
+	fsmonitor_batch__free_list(batch);
+	string_list_clear(&cookie_list, 0);
+}
+
+/*
+ * Non-blocking read of the inotify events stream. The inotify fd is polled
+ * frequently to help minimize the number of queue overflows.
+ */
+void fsm_listen__loop(struct fsmonitor_daemon_state *state)
+{
+	int poll_num;
+	const int interval = 1000;
+	time_t checked = time(NULL);
+	struct pollfd fds[1];
+
+	fds[0].fd = state->listen_data->fd_inotify;
+	fds[0].events = POLLIN;
+
+	/*
+	 * Our fs event listener is now running, so it's safe to start
+	 * serving client requests.
+	 */
+	ipc_server_start_async(state->ipc_server_data);
+
+	for (;;) {
+		switch (state->listen_data->shutdown) {
+		case SHUTDOWN_CONTINUE:
+			poll_num = poll(fds, 1, 1);
+			if (poll_num == -1) {
+				if (errno == EINTR)
+					continue;
+				error_errno(_("polling inotify message stream failed"));
+				state->listen_data->shutdown = SHUTDOWN_ERROR;
+				continue;
+			}
+
+			if ((time(NULL) - checked) >= interval) {
+				checked = time(NULL);
+				if (check_stale_dir_renames(&state->listen_data->renames,
+							    checked - interval)) {
+					trace_printf_key(&trace_fsmonitor,
+							 "Missed IN_MOVED_TO events, forcing shutdown");
+					state->listen_data->shutdown = SHUTDOWN_FORCE;
+					continue;
+				}
+			}
+
+			if (poll_num > 0 && (fds[0].revents & POLLIN))
+				handle_events(state);
+
+			continue;
+		case SHUTDOWN_ERROR:
+			state->listen_error_code = -1;
+			ipc_server_stop_async(state->ipc_server_data);
+			break;
+		case SHUTDOWN_FORCE:
+			state->listen_error_code = 0;
+			ipc_server_stop_async(state->ipc_server_data);
+			break;
+		case SHUTDOWN_STOP:
+		default:
+			state->listen_error_code = 0;
+			break;
+		}
+		return;
+	}
+}

compat/fsmonitor/fsm-path-utils-linux.c

@@ -0,0 +1,223 @@
+#include "git-compat-util.h"
+#include "fsmonitor-ll.h"
+#include "fsmonitor-path-utils.h"
+#include "gettext.h"
+#include "trace.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/statfs.h>
+
+#ifdef HAVE_LINUX_MAGIC_H
+#include <linux/magic.h>
+#endif
+
+/*
+ * Filesystem magic numbers for remote filesystems.
+ * Defined here if not available in linux/magic.h.
+ */
+#ifndef CIFS_SUPER_MAGIC
+#define CIFS_SUPER_MAGIC 0xff534d42
+#endif
+#ifndef SMB_SUPER_MAGIC
+#define SMB_SUPER_MAGIC 0x517b
+#endif
+#ifndef SMB2_SUPER_MAGIC
+#define SMB2_SUPER_MAGIC 0xfe534d42
+#endif
+#ifndef NFS_SUPER_MAGIC
+#define NFS_SUPER_MAGIC 0x6969
+#endif
+#ifndef AFS_SUPER_MAGIC
+#define AFS_SUPER_MAGIC 0x5346414f
+#endif
+#ifndef CODA_SUPER_MAGIC
+#define CODA_SUPER_MAGIC 0x73757245
+#endif
+#ifndef V9FS_MAGIC
+#define V9FS_MAGIC 0x01021997
+#endif
+#ifndef FUSE_SUPER_MAGIC
+#define FUSE_SUPER_MAGIC 0x65735546
+#endif
+
+/*
+ * Check if filesystem type is a remote filesystem.
+ */
+static int is_remote_fs(unsigned long f_type)
+{
+	switch (f_type) {
+	case CIFS_SUPER_MAGIC:
+	case SMB_SUPER_MAGIC:
+	case SMB2_SUPER_MAGIC:
+	case NFS_SUPER_MAGIC:
+	case AFS_SUPER_MAGIC:
+	case CODA_SUPER_MAGIC:
+	case FUSE_SUPER_MAGIC:
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * Get the filesystem type name for logging purposes.
+ */
+static const char *get_fs_typename(unsigned long f_type)
+{
+	switch (f_type) {
+	case CIFS_SUPER_MAGIC:
+		return "cifs";
+	case SMB_SUPER_MAGIC:
+		return "smb";
+	case SMB2_SUPER_MAGIC:
+		return "smb2";
+	case NFS_SUPER_MAGIC:
+		return "nfs";
+	case AFS_SUPER_MAGIC:
+		return "afs";
+	case CODA_SUPER_MAGIC:
+		return "coda";
+	case V9FS_MAGIC:
+		return "9p";
+	case FUSE_SUPER_MAGIC:
+		return "fuse";
+	default:
+		return "unknown";
+	}
+}
+
+/*
+ * Find the mount point for a given path by reading /proc/mounts.
+ * Returns the filesystem type for the longest matching mount point.
+ */
+static char *find_mount(const char *path, struct statfs *fs)
+{
+	FILE *fp;
+	struct strbuf line = STRBUF_INIT;
+	struct strbuf match = STRBUF_INIT;
+	struct strbuf fstype = STRBUF_INIT;
+	char *result = NULL;
+	struct statfs path_fs;
+
+	if (statfs(path, &path_fs) < 0)
+		return NULL;
+
+	fp = fopen("/proc/mounts", "r");
+	if (!fp)
+		return NULL;
+
+	while (strbuf_getline(&line, fp) != EOF) {
+		char *fields[6];
+		char *p = line.buf;
+		int i;
+
+		/* Parse mount entry: device mountpoint fstype options dump pass */
+		for (i = 0; i < 6 && p; i++) {
+			fields[i] = p;
+			p = strchr(p, ' ');
+			if (p)
+				*p++ = '\0';
+		}
+
+		if (i >= 3) {
+			const char *mountpoint = fields[1];
+			const char *type = fields[2];
+			struct statfs mount_fs;
+
+			/* Check if this mount point is a prefix of our path */
+			if (starts_with(path, mountpoint) &&
+			    (path[strlen(mountpoint)] == '/' ||
+			     path[strlen(mountpoint)] == '\0')) {
+				/* Check if filesystem ID matches */
+				if (statfs(mountpoint, &mount_fs) == 0 &&
+				    !memcmp(&mount_fs.f_fsid, &path_fs.f_fsid,
+					    sizeof(mount_fs.f_fsid))) {
+					/* Keep the longest matching mount point */
+					if (strlen(mountpoint) > match.len) {
+						strbuf_reset(&match);
+						strbuf_addstr(&match, mountpoint);
+						strbuf_reset(&fstype);
+						strbuf_addstr(&fstype, type);
+						*fs = mount_fs;
+					}
+				}
+			}
+		}
+	}
+
+	fclose(fp);
+	strbuf_release(&line);
+	strbuf_release(&match);
+
+	if (fstype.len)
+		result = strbuf_detach(&fstype, NULL);
+	else
+		strbuf_release(&fstype);
+
+	return result;
+}
+
+int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info)
+{
+	struct statfs fs;
+
+	if (statfs(path, &fs) == -1) {
+		int saved_errno = errno;
+		trace_printf_key(&trace_fsmonitor, "statfs('%s') failed: %s",
+				 path, strerror(saved_errno));
+		errno = saved_errno;
+		return -1;
+	}
+
+	trace_printf_key(&trace_fsmonitor,
+			 "statfs('%s') [type 0x%08lx]",
+			 path, (unsigned long)fs.f_type);
+
+	fs_info->is_remote = is_remote_fs(fs.f_type);
+
+	/*
+	 * Try to get filesystem type from /proc/mounts for a more
+	 * descriptive name.
+	 */
+	fs_info->typename = find_mount(path, &fs);
+	if (!fs_info->typename)
+		fs_info->typename = xstrdup(get_fs_typename(fs.f_type));
+
+	trace_printf_key(&trace_fsmonitor,
+			 "'%s' is_remote: %d, typename: %s",
+			 path, fs_info->is_remote, fs_info->typename);
+
+	return 0;
+}
+
+int fsmonitor__is_fs_remote(const char *path)
+{
+	struct fs_info fs;
+
+	if (fsmonitor__get_fs_info(path, &fs))
+		return -1;
+
+	free(fs.typename);
+
+	return fs.is_remote;
+}
+
+/*
+ * No-op for Linux - we don't have firmlinks like macOS.
+ */
+int fsmonitor__get_alias(const char *path UNUSED,
+			 struct alias_info *info UNUSED)
+{
+	return 0;
+}
+
+/*
+ * No-op for Linux - we don't have firmlinks like macOS.
+ */
+char *fsmonitor__resolve_alias(const char *path UNUSED,
+			       const struct alias_info *info UNUSED)
+{
+	return NULL;
+}

compat/fsmonitor/fsm-settings-linux.c

@@ -0,0 +1,71 @@
+#include "git-compat-util.h"
+#include "config.h"
+#include "fsmonitor-ll.h"
+#include "fsmonitor-ipc.h"
+#include "fsmonitor-settings.h"
+#include "fsmonitor-path-utils.h"
+
+#include <libgen.h>
+
+/*
+ * For the builtin FSMonitor, we create the Unix domain socket for the
+ * IPC in the .git directory.  If the working directory is remote,
+ * then the socket will be created on the remote file system.  This
+ * can fail if the remote file system does not support UDS file types
+ * (e.g. smbfs to a Windows server) or if the remote kernel does not
+ * allow a non-local process to bind() the socket.  (These problems
+ * could be fixed by moving the UDS out of the .git directory and to a
+ * well-known local directory on the client machine, but care should
+ * be taken to ensure that $HOME is actually local and not a managed
+ * file share.)
+ *
+ * FAT32 and NTFS working directories are problematic too.
+ *
+ * The builtin FSMonitor uses a Unix domain socket in the .git
+ * directory for IPC.  These Windows drive formats do not support
+ * Unix domain sockets, so mark them as incompatible for the daemon.
+ */
+static enum fsmonitor_reason check_uds_volume(struct repository *r)
+{
+	struct fs_info fs;
+	const char *ipc_path = fsmonitor_ipc__get_path(r);
+	char *path;
+	char *dir;
+
+	/*
+	 * Create a copy for dirname() since it may modify its argument.
+	 */
+	path = xstrdup(ipc_path);
+	dir = dirname(path);
+
+	if (fsmonitor__get_fs_info(dir, &fs) == -1) {
+		free(path);
+		return FSMONITOR_REASON_ERROR;
+	}
+
+	free(path);
+
+	if (fs.is_remote ||
+	    !strcmp(fs.typename, "msdos") ||
+	    !strcmp(fs.typename, "ntfs") ||
+	    !strcmp(fs.typename, "vfat")) {
+		free(fs.typename);
+		return FSMONITOR_REASON_NOSOCKETS;
+	}
+
+	free(fs.typename);
+	return FSMONITOR_REASON_OK;
+}
+
+enum fsmonitor_reason fsm_os__incompatible(struct repository *r, int ipc)
+{
+	enum fsmonitor_reason reason;
+
+	if (ipc) {
+		reason = check_uds_volume(r);
+		if (reason != FSMONITOR_REASON_OK)
+			return reason;
+	}
+
+	return FSMONITOR_REASON_OK;
+}

config.mak.uname

@@ -68,6 +68,16 @@ ifeq ($(uname_S),Linux)
 		BASIC_CFLAGS += -std=c99
         endif
 	LINK_FUZZ_PROGRAMS = YesPlease
+
+	# The builtin FSMonitor on Linux builds upon Simple-IPC.  Both require
+	# Unix domain sockets and PThreads.
+        ifndef NO_PTHREADS
+        ifndef NO_UNIX_SOCKETS
+	FSMONITOR_DAEMON_BACKEND = linux
+	FSMONITOR_OS_SETTINGS = linux
+	BASIC_CFLAGS += -DHAVE_LINUX_MAGIC_H
+        endif
+        endif
 endif
 ifeq ($(uname_S),GNU/kFreeBSD)
 	HAVE_ALLOCA_H = YesPlease

contrib/buildsystems/CMakeLists.txt

@@ -308,6 +308,16 @@ if(SUPPORTS_SIMPLE_IPC)
 
 		add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS)
 		list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-darwin.c)
+	elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+		add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND)
+		add_compile_definitions(HAVE_LINUX_MAGIC_H)
+		list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-linux.c)
+		list(APPEND compat_SOURCES compat/fsmonitor/fsm-health-linux.c)
+		list(APPEND compat_SOURCES compat/fsmonitor/fsm-ipc-linux.c)
+		list(APPEND compat_SOURCES compat/fsmonitor/fsm-path-utils-linux.c)
+
+		add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS)
+		list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-linux.c)
 	endif()
 endif()
 

t/t7527-builtin-fsmonitor.sh

@@ -520,6 +520,28 @@ test_expect_success 'directory changes to a file' '
 	grep "^event: dir1$" .git/trace
 '
 
+test_expect_success 'rapid nested directory creation' '
+	test_when_finished "git fsmonitor--daemon stop; rm -rf rapid" &&
+
+	start_daemon --tf "$PWD/.git/trace" &&
+
+	# Rapidly create nested directories to exercise race conditions
+	# where directory watches may be added concurrently during
+	# event processing and recursive scanning.
+	for i in $(test_seq 1 20)
+	do
+		mkdir -p "rapid/nested/dir$i/subdir/deep" || return 1
+	done &&
+
+	# Give the daemon time to process all events
+	sleep 1 &&
+
+	test-tool fsmonitor-client query --token 0 &&
+
+	# Verify daemon is still running (did not crash)
+	git fsmonitor--daemon status
+'
+
 # The next few test cases exercise the token-resync code.  When filesystem
 # drops events (because of filesystem velocity or because the daemon isn't
 # polling fast enough), we need to discard the cached data (relative to the