kbuild: Make dependencies at compile time

Making dependencies once up front is not ideal. For one, you don't need
them initially, since when you don't have the .o file, you bet you have
to build it no matter what the dependencies say - dependencies are about
deciding when to *re*build.

There's more reasons, like:
o you don't even know which files you'll build, so you have to go over
  all files (even over drivers/{sbus,s390,...} on i386)
o generated files don't exist yet, so you cannot pick up dependencies
  on them
o even if dependencies are right initially, they change when you work on 
  your tree or patch it, and nobody will notice unless you run "make dep"
  explicitly again

Anyway, gcc knows hows to emit a correct dependency list, so we just use
that. Well, a little bit of hacking is necessary to remove the dependency
on autoconf.h and put in individual CONFIG_WHAT_EVER dependencies instead,
since otherwise changing one config option would cause everything to be
rebuilt.

I should add that I didn't come up with this all by myself, most work
is actually done in gcc and there were discussions about using -MD on
kbuild-devel way back, so I should mention Keith Owens and Michael
Elizabeth Chastain, and probably others that I forgot, so I apologize
just in case.

Makefile

@@ -300,39 +300,41 @@ include/linux/version.h: Makefile
 # Helpers built in scripts/
 # ---------------------------------------------------------------------------
 
-scripts/mkdep scripts/split-include : scripts ;
+scripts/fixdep scripts/split-include : scripts ;
 
 .PHONY: scripts
 scripts:
 	@$(MAKE) -C scripts
 
-# Generate dependencies
+# Generate module versions
 # ---------------------------------------------------------------------------
 
-# 	In the same pass, generate module versions, that's why it's
-#	all mixed up here.
+# 	The targets are still named depend / dep for traditional
+#	reasons, but the only thing we do here is generating
+#	the module version checksums.
+#	FIXME: For now, we are also calling "archdep" from here,
+#	which should be replaced by a more sensible solution.
 
 .PHONY: depend dep $(patsubst %,_sfdep_%,$(SUBDIRS))
 
 depend dep: .hdepend
 
-#	.hdepend is missing prerequisites - in fact dependencies need
-#	to be redone basically each time anything changed - since
-#	that's too expensive, we traditionally rely on the user to
-#	run "make dep" manually whenever necessary. In this case,
-#	we make "FORCE" a prequisite, to force redoing the
-#	dependencies. Yeah, that's ugly, and it'll go away soon.
+ifdef CONFIG_MODVERSIONS
+
+#	Before descending for the actual build, we need module
+#	versions done. - Still using the old, illogical name
+#	.hdepend
 
-quiet_cmd_depend = Making dependencies (include)
-cmd_depend       = scripts/mkdep -- `find $(FINDHPATH) -name SCCS -prune -o -follow -name \*.h ! -name modversions.h -print` > $@
+#	.hdepend only indicates if we have generated module
+#	version checksums before now. For now, if they've
+#	been generated once, no rechecking will be done unless
+#	explicitly asked for using "make dep".
 
-.hdepend: scripts/mkdep include/linux/version.h include/asm \
+.hdepend: scripts/fixdep include/linux/version.h include/asm \
 	  $(if $(filter dep depend,$(MAKECMDGOALS)),FORCE)
-	$(call cmd,cmd_depend)
+	touch $@
 	@$(MAKE) $(patsubst %,_sfdep_%,$(SUBDIRS))
-ifdef CONFIG_MODVERSIONS
 	@$(MAKE) include/linux/modversions.h
-endif
 	@$(MAKE) archdep
 
 $(patsubst %,_sfdep_%,$(SUBDIRS)): FORCE
@@ -358,6 +360,15 @@ include/linux/modversions.h: FORCE
 		mv -f $@.tmp $@; \
 	fi
 
+else # !CONFIG_MODVERSIONS
+
+.hdepend: include/linux/version.h include/asm \
+	  $(if $(filter dep depend,$(MAKECMDGOALS)),FORCE)
+	touch $@
+	@$(MAKE) archdep
+
+endif # CONFIG_MODVERSIONS
+
 # ---------------------------------------------------------------------------
 # Modules
 
@@ -592,8 +603,8 @@ MRPROPER_FILES += \
 	scripts/lxdialog/*.o scripts/lxdialog/lxdialog \
 	.menuconfig.log \
 	include/asm \
-	.hdepend scripts/mkdep scripts/split-include scripts/docproc \
-	$(TOPDIR)/include/linux/modversions.h \
+	.hdepend scripts/split-include scripts/docproc \
+	scripts/fixdep $(TOPDIR)/include/linux/modversions.h \
 	tags TAGS kernel.spec \
 
 # 	directories removed with 'make mrproper'
@@ -607,7 +618,8 @@ include arch/$(ARCH)/Makefile
 
 clean:	archclean
 	@echo 'Cleaning up'
-	@find . \( -name \*.[oas] -o -name core -o -name .\*.cmd \) -type f -print \
+	@find . \( -name \*.[oas] -o -name core -o -name .\*.cmd -o \
+		   -name .\*.tmp -o -name .\*.d \) -type f -print \
 		| grep -v lxdialog/ | xargs rm -f
 	@rm -f $(CLEAN_FILES)
 	@rm -rf $(CLEAN_DIRS)

Rules.make

@@ -96,7 +96,7 @@ first_rule: vmlinux $(if $(BUILD_MODULES),$(obj-m))
 # Compile C sources (.c)
 # ---------------------------------------------------------------------------
 
-# FIXME: if we don't know if built-in or modular, assume built-in.
+# If we don't know if built-in or modular, assume built-in.
 # Only happens in Makefiles which override the default first_rule:
 modkern_cflags := $(CFLAGS_KERNEL)
 
@@ -120,25 +120,25 @@ quiet_cmd_cc_s_c = CC     $(RELDIR)/$@
 cmd_cc_s_c       = $(CC) $(c_flags) -S -o $@ $< 
 
 %.s: %.c FORCE
-	$(call if_changed,cmd_cc_s_c)
+	$(call cmd,cmd_cc_s_c)
 
 quiet_cmd_cc_i_c = CPP    $(RELDIR)/$@
 cmd_cc_i_c       = $(CPP) $(c_flags)   -o $@ $<
 
 %.i: %.c FORCE
-	$(call if_changed,cmd_cc_i_c)
+	$(call cmd,cmd_cc_i_c)
 
 quiet_cmd_cc_o_c = CC     $(RELDIR)/$@
-cmd_cc_o_c       = $(CC) $(c_flags) -c -o $@ $<
+cmd_cc_o_c       = $(CC) -Wp,-MD,.$(subst /,_,$@).d $(c_flags) -c -o $@ $<
 
 %.o: %.c FORCE
-	$(call if_changed,cmd_cc_o_c)
+	$(call if_changed_dep,cc_o_c)
 
 quiet_cmd_cc_lst_c = Generating $(RELDIR)/$@
 cmd_cc_lst_c     = $(CC) $(c_flags) -g -c -o $*.o $< && $(TOPDIR)/scripts/makelst $*.o $(TOPDIR)/System.map $(OBJDUMP) > $@
 
 %.lst: %.c FORCE
-	$(call if_changed,cmd_cc_lst_c)
+	$(call cmd,cmd_cc_lst_c)
 
 # Compile assembler sources (.S)
 # ---------------------------------------------------------------------------
@@ -158,13 +158,13 @@ quiet_cmd_as_s_S = CPP    $(RELDIR)/$@
 cmd_as_s_S       = $(CPP) $(a_flags)   -o $@ $< 
 
 %.s: %.S FORCE
-	$(call if_changed,cmd_as_s_S)
+	$(call cmd,cmd_as_s_S)
 
 quiet_cmd_as_o_S = AS     $(RELDIR)/$@
-cmd_as_o_S       = $(CC) $(a_flags) -c -o $@ $<
+cmd_as_o_S       = $(CC) -Wp,-MD,.$(subst /,_,$@).d $(a_flags) -c -o $@ $<
 
 %.o: %.S FORCE
-	$(call if_changed,cmd_as_o_S)
+	$(call if_changed_dep,as_o_S)
 
 # If a Makefile does define neither O_TARGET nor L_TARGET,
 # use a standard O_TARGET named "built-in.o"
@@ -226,14 +226,10 @@ $(multi-used-m) : %.o: $(multi-objs-m) FORCE
 	$(call if_changed,cmd_link_multi)
 
 #
-# This make dependencies quickly
+# This makes module versions
 #
 
-quiet_cmd_fastdep = Making dependencies ($(RELDIR))
-cmd_fastdep       = $(TOPDIR)/scripts/mkdep $(CFLAGS) $(EXTRA_CFLAGS) -- $(wildcard *.[chS]) > .depend
-
 fastdep: FORCE
-	$(call cmd,cmd_fastdep)
 ifdef ALL_SUB_DIRS
 	@$(MAKE) $(patsubst %,_sfdep_%,$(ALL_SUB_DIRS)) _FASTDEP_ALL_SUB_DIRS="$(ALL_SUB_DIRS)"
 endif
@@ -287,7 +283,9 @@ modules_install: _modinst__ $(patsubst %,_modinst_%,$(MOD_SUB_DIRS))
 
 # Add FORCE to the prequisites of a target to force it to be always rebuilt.
 # ---------------------------------------------------------------------------
+
 .PHONY: FORCE
+
 FORCE:
 
 #
@@ -301,7 +299,6 @@ script:
 # Separate the object into "normal" objects and "exporting" objects
 # Exporting objects are: all objects that define symbol tables
 #
-ifdef CONFIG_MODULES
 
 ifdef CONFIG_MODVERSIONS
 ifneq "$(strip $(export-objs))" ""
@@ -323,16 +320,17 @@ endif
 # We don't track dependencies for .ver files, so we FORCE to check
 # them always (i.e. always at "make dep" time).
 
+quiet_cmd_create_ver = Creating $@
 cmd_create_ver = $(CC) $(CFLAGS) $(EXTRA_CFLAGS) -E -D__GENKSYMS__ $< | \
 		 $(GENKSYMS) $(genksyms_smp_prefix) -k $(VERSION).$(PATCHLEVEL).$(SUBLEVEL) > $@.tmp
 
 $(MODINCL)/$(MODPREFIX)%.ver: %.c FORCE
 	@echo $(cmd_create_ver)
-	@$(cmd_create_ver)
+	@$(call cmd,cmd_create_ver)
 	@if [ -r $@ ] && cmp -s $@ $@.tmp; then \
 	  echo $@ is unchanged; rm -f $@.tmp; \
 	else \
-	  echo mv $@.tmp $@; mv -f $@.tmp $@; \
+	  mv -f $@.tmp $@; \
 	fi
 
 # updates .ver files but not modversions.h
@@ -340,30 +338,8 @@ fastdep: $(addprefix $(MODINCL)/$(MODPREFIX),$(export-objs:.o=.ver))
 
 endif # export-objs 
 
-# make dep cannot correctly figure out the dependency on the generated
-# modversions.h, so we list them here:
-# o files which export symbols and are compiled into the kernel include
-#   it (to generate a correct symbol table)
-# o all modules get compiled with -include modversions.h
-
-$(filter $(export-objs),$(real-objs-y)): $(TOPDIR)/include/linux/modversions.h
-$(real-objs-m): $(TOPDIR)/include/linux/modversions.h
-
 endif # CONFIG_MODVERSIONS
 
-endif # CONFIG_MODULES
-
-#
-# include dependency files if they exist
-#
-ifneq ($(wildcard .depend),)
-include .depend
-endif
-
-ifneq ($(wildcard $(TOPDIR)/.hdepend),)
-include $(TOPDIR)/.hdepend
-endif
-
 # ---------------------------------------------------------------------------
 # Check if command line has changed
 
@@ -395,7 +371,7 @@ endif
 #   which is saved in .<target>.o, to the current command line using
 #   the two filter-out commands)
 
-# read all saved command lines
+# read all saved command lines and dependencies
 
 cmd_files := $(wildcard .*.cmd)
 ifneq ($(cmd_files),)
@@ -409,6 +385,20 @@ if_changed = $(if $(strip $? \
 			  $(filter-out $(cmd_$(@F)),$($(1)))),\
 	       @$(if $($(quiet)$(1)),echo '  $($(quiet)$(1))' &&) $($(1)) && echo 'cmd_$@ := $($(1))' > $(@D)/.$(@F).cmd)
 
+
+# execute the command and also postprocess generated .d dependencies
+# file
+
+if_changed_dep = $(if $(strip $? \
+		          $(filter-out $(cmd_$(1)),$(cmd_$@))\
+			  $(filter-out $(cmd_$@),$(cmd_$(1)))),\
+	@set -e; \
+	$(if $($(quiet)cmd_$(1)),echo '$($(quiet)cmd_$(1))';) \
+	$(cmd_$(1)); \
+	$(TOPDIR)/scripts/fixdep $(subst /,_,$@) $(TOPDIR) '$(cmd_$(1))' > .$(subst /,_,$@).tmp; \
+	rm -f .$(subst /,_,$@).d; \
+	mv -f .$(subst /,_,$@).tmp .$(subst /,_,$@).cmd )
+
 # If quiet is set, only print short version of command
 
 cmd = @$(if $($(quiet)$(1)),echo '  $($(quiet)$(1))' &&) $($(1))

scripts/Makefile

@@ -7,9 +7,9 @@
 # can't do it
 CHMOD_FILES := docgen gen-all-syms kernel-doc mkcompile_h mkversion_h makelst
 
-all: mkdep split-include $(CHMOD_FILES)
+all: fixdep split-include $(CHMOD_FILES)
 
-mkdep: mkdep.c
+fixdep: fixdep.c
 	$(HOSTCC) $(HOSTCFLAGS) -o $@ $<
 
 split-include: split-include.c

scripts/fixdep.c

+/*
+ * "Optimize" a list of dependencies as spit out by gcc -MD 
+ * for the kernel build
+ * ===========================================================================
+ *
+ * Author       Kai Germaschewski
+ * Copyright    2002 by Kai Germaschewski  <kai.germaschewski@gmx.de>
+ *
+ * This software may be used and distributed according to the terms
+ * of the GNU General Public License, incorporated herein by reference.
+ *
+ *
+ * Introduction:
+ * 
+ * gcc produces a very nice and correct list of dependencies which
+ * tells make when to remake a file.
+ *
+ * To use this list as-is however has the drawback that virtually
+ * every file in the kernel includes <linux/config.h> which then again
+ * includes <linux/autoconf.h>
+ *
+ * If the user re-runs make *config, linux/autoconf.h will be
+ * regenerated.  make notices that and will rebuild every file which
+ * includes autconf.h, i.e. basically all files. This is extremely
+ * annoying if the user just changed CONFIG_HIS_DRIVER from n to m.
+ * 
+ * So we play the same trick that "mkdep" played before. We replace
+ * the dependency on linux/autoconf.h by a dependency on every config
+ * option which is mentioned in any of the listed prequisites.
+ *  
+ * To be exact, split-include populates a tree in include/config/,
+ * e.g. include/config/his/driver.h, which contains the #define/#undef
+ * for the CONFIG_HIS_DRIVER option.
+ *
+ * So if the user changes his CONFIG_HIS_DRIVER option, only the objects
+ * which depend on "include/linux/config/his/driver.h" will be rebuilt,
+ * so most likely only his driver ;-) 
+ *
+ * The idea above dates, by the way, back to Michael E Chastain, AFAIK.
+ * 
+ * So to get dependencies right, there two issues:
+ * o if any of the files the compiler read changed, we need to rebuild
+ * o if the command line given to the compile the file changed, we
+ *   better rebuild as well.
+ *
+ * The former is handled by using the -MD output, the later by saving
+ * the command line used to compile the old object and comparing it
+ * to the one we would now use.
+ *
+ * Again, also this idea is pretty old and has been discussed on
+ * kbuild-devel a long time ago. I don't have a sensibly working
+ * internet connection right now, so I rather don't mention names
+ * without double checking.
+ *
+ * This code here has been based partially based on mkdep.c, which
+ * says the following about its history:
+ *
+ *   Copyright abandoned, Michael Chastain, <mailto:mec@shout.net>.
+ *   This is a C version of syncdep.pl by Werner Almesberger.
+ *
+ *
+ * It is invoked as
+ *
+ *   fixdep <target> <topdir> <cmdline>
+ *
+ * and will read the dependency file ".<target>.d".
+ *
+ * The transformed dependency snipped is written to stdout.
+ *
+ * It first generates a line
+ *
+ *   cmd_<target> = <cmdline>
+ *
+ * and then basically copies the .<target>.d file to stdout, in the
+ * process filtering out the dependency on linux/autconf.h and adding
+ * dependencies on include/config/my/option.h for every
+ * CONFIG_MY_OPTION encountered in any of the prequisites.
+ *
+ * It will also filter out all the dependencies on *.ver. We need
+ * to make sure that the generated version checksum are globally up
+ * to date before even starting the recursive build, so it's too late
+ * at this point anyway.
+ *
+ * The algorithm to grep for "CONFIG_..." is bit unusual, but should
+ * be fast ;-) We don't even try to really parse the header files, but
+ * merely grep, i.e. if CONFIG_FOO is mentioned in a comment, it will
+ * be picked up as well. It's not a problem with respect to
+ * correctness, since that can only give too many dependencies, thus
+ * we cannot miss a rebuild. Since people tend to not mention totally
+ * unrelated CONFIG_ options all over the place, it's not an
+ * efficiency problem either.
+ * 
+ * (Note: it'd be easy to port over the complete mkdep state machine,
+ *  but I don't think the added complexity is worth it)
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <limits.h>
+#include <ctype.h>
+#include <netinet/in.h>
+
+#define INT_CONF ntohl(0x434f4e46)
+#define INT_ONFI ntohl(0x4f4e4649)
+#define INT_NFIG ntohl(0x4e464947)
+#define INT_FIG_ ntohl(0x4649475f)
+
+char *topdir;
+
+void usage(void)
+
+{
+	fprintf(stderr, "Usage: fixdep <target> <topdir> <cmdline>\n");
+	exit(1);
+}
+
+void print_cmdline(char *target, char *cmdline)
+{
+	char *s = strdup(target);
+	char *p = s;
+
+	if (!s) {
+		fprintf(stderr, "no mem!\n");
+		exit(2);
+	}
+	while ((p = strchr(p,'/')))
+		*p = '_';
+
+	printf("cmd_%s := %s\n\n", s, cmdline);
+
+	free(s);
+}
+
+char * str_config  = NULL;
+int    size_config = 0;
+int    len_config  = 0;
+
+/*
+ * Grow the configuration string to a desired length.
+ * Usually the first growth is plenty.
+ */
+void grow_config(int len)
+{
+	while (len_config + len > size_config) {
+		if (size_config == 0)
+			size_config = 2048;
+		str_config = realloc(str_config, size_config *= 2);
+		if (str_config == NULL)
+			{ perror("malloc"); exit(1); }
+	}
+}
+
+
+
+/*
+ * Lookup a value in the configuration string.
+ */
+int is_defined_config(const char * name, int len)
+{
+	const char * pconfig;
+	const char * plast = str_config + len_config - len;
+	for ( pconfig = str_config + 1; pconfig < plast; pconfig++ ) {
+		if (pconfig[ -1] == '\n'
+		&&  pconfig[len] == '\n'
+		&&  !memcmp(pconfig, name, len))
+			return 1;
+	}
+	return 0;
+}
+
+/*
+ * Add a new value to the configuration string.
+ */
+void define_config(const char * name, int len)
+{
+	grow_config(len + 1);
+
+	memcpy(str_config+len_config, name, len);
+	len_config += len;
+	str_config[len_config++] = '\n';
+}
+
+/*
+ * Clear the set of configuration strings.
+ */
+void clear_config(void)
+{
+	len_config = 0;
+	define_config("", 0);
+}
+
+/*
+ * Record the use of a CONFIG_* word.
+ */
+void use_config(char *m, int slen)
+{
+	char s[PATH_MAX];
+	char *p;
+
+	if (is_defined_config(m, slen))
+	    return;
+
+	define_config(m, slen);
+
+	memcpy(s, m, slen); s[slen] = 0;
+
+	for (p = s; p < s + slen; p++) {
+		if (*p == '_')
+			*p = '/';
+		else
+			*p = tolower(*p);
+	}
+	printf("    $(wildcard %s/include/config/%s.h) \\\n", topdir, s);
+}
+
+void parse_config_file(char *map, size_t len)
+{
+	int *end = (int *) (map + len);
+	// start at +1, so that p can never be < map
+	int *m   = (int *) map + 1;
+	char *p, *q;
+
+	for (; m < end; m++) {
+		if (*m == INT_CONF) { p = (char *) m  ; goto conf; }
+		if (*m == INT_ONFI) { p = (char *) m-1; goto conf; }
+		if (*m == INT_NFIG) { p = (char *) m-2; goto conf; }
+		if (*m == INT_FIG_) { p = (char *) m-3; goto conf; }
+		continue;
+	conf:
+		if (p > map + len - 7)
+			continue;
+		if (memcmp(p, "CONFIG_", 7))
+			continue;
+		for (q = p + 7; q < map + len; q++) {
+			if (!(isalnum(*q)))
+				goto found;
+		}
+		continue;
+
+	found: 
+		use_config(p+7, q-p-7);
+	}
+}
+
+/* test is s ends in sub */
+int strrcmp(char *s, char *sub)
+{
+	int slen = strlen(s);
+	int sublen = strlen(sub);
+  
+	if (sublen > slen)
+		return 1;
+	
+	return memcmp(s + slen - sublen, sub, sublen);
+}
+
+void do_config_file(char *filename)
+{
+	struct stat st;
+	int fd;
+	void *map;
+
+	fd = open(filename, O_RDONLY);
+	if (fd < 0) {
+		perror(filename);
+		exit(2);
+	}
+	fstat(fd, &st);
+	if (st.st_size == 0) {
+		close(fd);
+		return;
+	}
+	map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if ((long) map == -1) {
+		perror("mmap");
+		close(fd);
+		return;
+	}
+	
+	parse_config_file(map, st.st_size);
+
+	munmap(map, st.st_size);
+}
+
+void parse_dep_file(void *map, size_t len)
+{
+	char *m = map;
+	char *end = map + len;
+	char *p;
+	char s[PATH_MAX];
+
+	p = strchr(m, ':');
+	if (!p) {
+		fprintf(stderr, "parse error at %d", __LINE__);
+		exit(1);
+	}
+	memcpy(s, m, p-m); s[p-m] = 0;
+	printf("%s: \\\n", s);
+	m = p+1;
+
+	clear_config();
+
+	while (m < end) {
+		while (*m == ' ' || *m == '\\' || *m == '\n')
+			m++;
+
+		p = strchr(m, ' ');
+		if (!p) {
+			p = end;
+			while (!isalpha(*p)) p--;
+			p++;
+		}
+		memcpy(s, m, p-m); s[p-m] = 0;
+		if (strrcmp(s, "include/linux/autoconf.h") &&
+		    strrcmp(s, ".ver")) {
+			printf("  %s \\\n", s);
+			do_config_file(s);
+		}
+		m = p + 1;
+	}
+	printf("\n");
+}
+
+void print_deps(char *target)
+{
+	char filename[PATH_MAX];
+	struct stat st;
+	int fd;
+	void *map;
+
+	sprintf(filename, ".%s.d", target);
+	fd = open(filename, O_RDONLY);
+	if (fd < 0) {
+		perror(filename);
+		exit(2);
+	}
+	fstat(fd, &st);
+	if (st.st_size == 0) {
+		fprintf(stderr,"%s is empty\n",filename);
+		close(fd);
+		return;
+	}
+	map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if ((long) map == -1) {
+		perror("mmap");
+		close(fd);
+		return;
+	}
+	
+	parse_dep_file(map, st.st_size);
+
+	munmap(map, st.st_size);
+}
+
+void traps(void)
+{
+	char *test = "CONF";
+
+	if (*(int *)test != INT_CONF) {
+		fprintf(stderr, "sizeof(int) != 4 or wrong endianess? %#x\n",
+			*(int *)test);
+		exit(2);
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	char *target, *cmdline;
+	
+	traps();
+
+	if (argc != 4)
+		usage();
+		
+	target = argv[1];
+	topdir = argv[2];
+	cmdline = argv[3];
+
+	print_cmdline(target, cmdline);
+	print_deps(target);
+
+	return 0;
+}