Added Matt Armstrong's header caching (based on an implementation by Craig McPheeters).

git-svn-id: file:///srv/svn/repos/haiku/trunk/current@1962 a95241bf-73f2-0310-859d-f6bbb57e9c96

src/tools/jam/Jambase

@@ -1218,6 +1218,13 @@ rule SubDir
 	LOCATE_SOURCE = $(ALL_LOCATE_TARGET) $(SUBDIR) ;
 	LOCATE_TARGET = $(ALL_LOCATE_TARGET) $(SUBDIR) ;
 	SOURCE_GRIST = [ FGrist $(<[2-]) ] ;
+	## LOCAL CHANGE -- OPT_HEADER_CACHE_EXT.  With the header
+	# cache, we can grist all files found during a header scan
+	# without incurring a performance penalty.
+	#
+	HDRGRIST = $(SOURCE_GRIST) ;
+	#
+	## LOCAL CHANGE
 
 	# Reset per-directory ccflags, hdrs
 
@@ -1311,31 +1318,40 @@ rule FGristFiles
 
 rule FGristSourceFiles
 {
-	# Produce source file name name with grist in it, 
+	## LOCAL CHANGE: OPT_HEADER_CACHE_EXT
-	# if SOURCE_GRIST is set.
+	# With header caching, there is no performance penalty to gristing
-
+	# header files.	 It is also not correct to assume that header
-	# Leave header files alone, because they have a global
+	# files have global visibility.
-	# visibility.
+	#
-
+	# Here we comment out the old version and replace it with the new.
-	if ! $(SOURCE_GRIST)
+#	# Produce source file name name with grist in it, 
-	{
+#	# if SOURCE_GRIST is set.
-	    return $(<) ;
+#
-	}
+#	# Leave header files alone, because they have a global
-	else 
+#	# visibility.
-	{
+#
-	    local _i _o ;
+#	if ! $(SOURCE_GRIST)
-
+#	{
-	    for _i in $(<)
+#	    return $(<) ;
-	    {
+#	}
-		switch $(_i)
+#	else 
-		{
+#	{
-		case *.h :	_o += $(_i) ;
+#	    local _i _o ;
-		case * : 	_o += $(_i:G=$(SOURCE_GRIST)) ;
+#
-		}
+#	    for _i in $(<)
-	    }
+#	    {
-
+#		switch $(_i)
-	    return $(_o) ;
+#		{
-	}
+#		case *.h :	_o += $(_i) ;
+#		case * : 	_o += $(_i:G=$(SOURCE_GRIST)) ;
+#		}
+#	    }
+#
+#	    return $(_o) ;
+#	}
+	return [ FGristFiles $(<) ] ;
+	#
+	## LOCAL CHANGE: end
 }
 
 rule FIsPrefix

src/tools/jam/Jamfile

@@ -49,6 +49,14 @@ if $(DEBUG)
 
 if $(OS) = NT 			{ CCFLAGS += /DNT ; }
 
+### LOCAL CHANGE
+# 
+# Include header caching.
+#
+DEFINES += OPT_HEADER_CACHE_EXT ;
+#
+### LOCAL CHANGE
+
 # Do we know yacc?
 
 if $(YACC) 	{ code += jamgram.y ; }
@@ -65,6 +73,15 @@ if $(YACC) && $(SUFEXE) = ""
 	GenFile jamgram.y jamgramtab.h : yyacc jamgram.yy ;
 }
 
+### LOCAL CHANGE
+# 
+# These files contain locally developed improvements.
+#
+# code primarily not written locally, but grabbed from the net
+code += hcache.c ;
+#
+### LOCAL CHANGE
+
 #
 # How to build the compiled in jambase.
 #
@@ -113,6 +130,12 @@ ALLSOURCE =
 	scan.h search.c search.h timestamp.c timestamp.h variable.c
 	variable.h yyacc ;
 
+### LOCAL CHANGE
+#
+ALLSOURCE += LOCAL_DIFFERENCES.txt ;
+#
+### LOCAL CHANGE
+
 rule Ball
 {
 	NotFile balls ;

src/tools/jam/hcache.c

@@ -0,0 +1,421 @@
+/*
+ * This file has been donated to Jam.
+ */
+
+# include "jam.h"
+# include "lists.h"
+# include "parse.h"
+# include "rules.h"
+# include "regexp.h"
+# include "headers.h"
+# include "newstr.h"
+# include "hash.h"
+# include "hcache.h"
+# include "variable.h"
+# include "search.h"
+
+#ifdef OPT_HEADER_CACHE_EXT
+
+/*
+ * Craig W. McPheeters, Alias|Wavefront.
+ *
+ * hcache.c hcache.h - handle cacheing of #includes in source files
+ *
+ * Create a cache of files scanned for headers.  When starting jam,
+ * look for the cache file and load it if present.  When finished the
+ * binding phase, create a new header cache.  The cache contains
+ * files, their timestamps and the header files found in their scan.
+ * During the binding phase of jam, look in the header cache first for
+ * the headers contained in a file.  If the cache is present and
+ * valid, use its contents.  This results in dramatic speedups with
+ * large projects (eg. 3min -> 1min startup for one project.)
+ *
+ * External routines:
+ *    hcache_init() - read and parse the local .jamdeps file.
+ *    hcache_done() - write a new .jamdeps file
+ *    hcache() - return list of headers on target.  Use cache or do a scan.
+ *    
+ * The dependency file format is an ascii file with 1 line per target.
+ * Each line has the following fields:
+ * @boundname@ timestamp @file@ @file@ @file@ ... \n
+ * */
+
+struct hcachedata {
+    char		*boundname;
+    time_t		time;
+    LIST		*includes;
+    LIST		*hdrscan; /* the HDRSCAN value for this target */
+    int			age;	/* if too old, we'll remove it from cache */
+    struct hcachedata	*next;
+} ;
+
+typedef struct hcachedata HCACHEDATA ;
+
+
+static struct hash *hcachehash = 0;
+static HCACHEDATA  *hcachelist = 0; 
+
+static int queries = 0;
+static int hits = 0;
+
+#define CACHE_FILE_VERSION "version 4"
+#define CACHE_RECORD_HEADER "header"
+#define CACHE_RECORD_END "end"
+
+/*
+ * Return the name of the header cache file.  May return NULL.
+ *
+ * The user sets this by setting the HCACHEFILE variable in a Jamfile.
+ * We cache the result so the user can't change the cache file during
+ * header scanning.
+ */
+static char*
+cache_name(void)
+{
+    static char* name = 0;
+    if (!name) {
+	LIST *hcachevar = var_get("HCACHEFILE");
+
+	if (hcachevar) {
+	    TARGET *t = bindtarget( hcachevar->string );
+
+	    pushsettings( t->settings );
+	    t->boundname = search( t->name, &t->time );
+	    popsettings( t->settings );
+
+	    if (hcachevar) {
+		name = copystr(t->boundname);
+	    }
+	}
+    }
+    return name;
+}
+
+/*
+ * Return the maximum age a cache entry can have before it is purged
+ * from the cache.
+ */
+static int
+cache_maxage(void)
+{
+    int age = 100;
+    LIST *var = var_get("HCACHEMAXAGE");
+
+    if (var) {
+	age = atoi(var->string);
+	if (age < 0)
+	    age = 0;
+    }
+
+    return age;
+}
+
+/*
+ * Read a netstring.  The caveat is that the string can't contain
+ * ASCII 0.  The returned value is as returned by newstr(), so it need
+ * not be freed.
+ */
+char*
+read_netstring(FILE* f)
+{
+    unsigned long len;
+    static char* buf = NULL;
+    static unsigned long buf_len = 0;
+
+    if (fscanf(f, " %9lu", &len) != 1)
+	return NULL;
+    if (fgetc(f) != (int)'\t')
+	return NULL;
+
+    if (len > 1024 * 64)
+	return NULL;		/* sanity check */
+
+    if (len > buf_len)
+    {
+	unsigned long new_len = buf_len * 2;
+	if (new_len < len)
+	    new_len = len;
+	buf = realloc(buf, new_len + 1);
+	if (buf)
+	    buf_len = new_len;
+    }
+
+    if (!buf)
+	return NULL;
+
+    if (fread(buf, 1, len, f) != len)
+	return NULL;
+    if (fgetc(f) != (int)'\n')
+	return NULL;
+
+    buf[len] = 0;
+    return newstr(buf);
+}
+
+/*
+ * Write a netstring.
+ */
+void
+write_netstring(FILE* f, const char* s)
+{
+    if (!s)
+	s = "";
+    fprintf(f, "%lu\t%s\n", strlen(s), s);
+}
+
+void
+hcache_init()
+{
+    HCACHEDATA  cachedata, *c;
+    FILE	*f;
+    char	*version;
+    int		header_count = 0;
+    char*	hcachename;
+
+    hcachehash = hashinit (sizeof (HCACHEDATA), "hcache");
+
+    if (! (hcachename = cache_name()))
+	return;
+
+    if (! (f = fopen (hcachename, "rb" )))
+	return;
+    
+    version = read_netstring(f);
+    if (!version || strcmp(version, CACHE_FILE_VERSION)) {
+	fclose(f);
+	return;
+    }
+
+    while (1)
+    {
+	char* record_type;
+	char *time_str;
+	char *age_str;
+	char *includes_count_str;
+	char *hdrscan_count_str;
+	int i, count;
+	LIST *l;
+
+	record_type = read_netstring(f);
+	if (!record_type) {
+	    fprintf(stderr, "invalid %s\n", hcachename);
+	    goto bail;
+	}
+	if (!strcmp(record_type, CACHE_RECORD_END)) {
+	    break;
+	}
+	if (strcmp(record_type, CACHE_RECORD_HEADER)) {
+	    fprintf(stderr, "invalid %s with record separator <%s>\n",
+		    hcachename, record_type ? record_type : "<null>");
+	    goto bail;
+	}
+	
+	c = &cachedata;
+	    
+	c->boundname = read_netstring(f);
+	time_str = read_netstring(f);
+	age_str = read_netstring(f);
+	includes_count_str = read_netstring(f);
+	
+	if (!c->boundname || !time_str || !age_str
+	    || !includes_count_str)
+	{
+	    fprintf(stderr, "invalid %s\n", hcachename);
+	    goto bail;
+	}
+
+	c->time = atoi(time_str);
+	c->age = atoi(age_str) + 1;
+
+	count = atoi(includes_count_str);
+	for (l = 0, i = 0; i < count; i++) {
+	    char* s = read_netstring(f);
+	    if (!s) {
+		fprintf(stderr, "invalid %s\n", hcachename);
+		goto bail;
+	    }
+	    l = list_new(l, s);
+	}
+	c->includes = l;
+
+	hdrscan_count_str = read_netstring(f);
+	if (!includes_count_str) {
+	    list_free(c->includes);
+	    fprintf(stderr, "invalid %s\n", hcachename);
+	    goto bail;
+	}
+
+	count = atoi(hdrscan_count_str);
+	for (l = 0, i = 0; i < count; i++) {
+	    char* s = read_netstring(f);
+	    if (!s) {
+		fprintf(stderr, "invalid %s\n", hcachename);
+		goto bail;
+	    }
+	    l = list_new(l, s);
+	}
+	c->hdrscan = l;
+
+	if (!hashenter(hcachehash, (HASHDATA **)&c)) {
+	    fprintf(stderr, "can't insert header cache item, bailing on %s\n",
+		    hcachename);
+	    goto bail;
+	}
+
+	c->next = hcachelist;
+	hcachelist = c;
+
+	header_count++;
+    }
+
+    if (DEBUG_HEADER) {
+	printf("hcache read from file %s\n", hcachename);
+    }
+    
+ bail:
+    fclose(f);
+}
+
+void
+hcache_done()
+{
+    FILE	*f;
+    HCACHEDATA  *c;
+    int 	header_count = 0;
+    char*	hcachename;
+    int		maxage;
+    
+    if (!hcachehash)
+	return;
+
+    if (! (hcachename = cache_name()))
+	return;
+
+    if (! (f = fopen (hcachename, "wb" )))
+	return;
+
+    maxage = cache_maxage();
+
+    /* print out the version */
+    write_netstring(f, CACHE_FILE_VERSION);
+
+    c = hcachelist;
+    for (c = hcachelist; c; c = c->next) {
+	LIST	*l;
+	char time_str[30];
+	char age_str[30];
+	char includes_count_str[30];
+	char hdrscan_count_str[30];
+
+	if (maxage == 0)
+	    c->age = 0;
+	else if (c->age > maxage)
+	    continue;
+
+	sprintf(includes_count_str, "%lu", list_length(c->includes));
+	sprintf(hdrscan_count_str, "%lu", list_length(c->hdrscan));
+	sprintf(time_str, "%lu", c->time);
+	sprintf(age_str, "%lu", c->age);
+
+	write_netstring(f, CACHE_RECORD_HEADER);
+	write_netstring(f, c->boundname);
+	write_netstring(f, time_str);
+	write_netstring(f, age_str);
+	write_netstring(f, includes_count_str);
+	for (l = c->includes; l; l = list_next(l)) {
+	    write_netstring(f, l->string);
+	}
+	write_netstring(f, hdrscan_count_str);
+	for (l = c->hdrscan; l; l = list_next(l)) {
+	    write_netstring(f, l->string);
+	}
+	fputs("\n", f);
+	header_count++;
+    }
+    write_netstring(f, CACHE_RECORD_END);
+
+    if (DEBUG_HEADER) {
+	printf("hcache written to %s.   %d dependencies, %.0f%% hit rate\n",
+	       hcachename, header_count,
+	       queries ? 100.0 * hits / queries : 0);
+    }
+
+    fclose (f);
+}
+
+LIST *
+hcache (TARGET *t, int rec, regexp *re[], LIST *hdrscan)
+{
+    HCACHEDATA  cachedata, *c = &cachedata;
+    LIST 	*l = 0;
+
+    ++queries;
+
+    c->boundname = t->boundname;
+
+    if (hashcheck (hcachehash, (HASHDATA **) &c))
+    {
+	if (c->time == t->time)
+	{
+	    LIST *l1 = hdrscan, *l2 = c->hdrscan;
+	    while (l1 && l2) {
+		if (l1->string != l2->string) {
+		    l1 = NULL;
+		} else {
+		    l1 = list_next(l1);
+		    l2 = list_next(l2);
+		}
+	    }
+	    if (l1 || l2) {
+		if (DEBUG_HEADER)
+		    printf("HDRSCAN out of date in cache for %s\n",
+			   t->boundname);
+
+		printf("HDRSCAN out of date for %s\n", t->boundname);
+		printf(" real  : ");
+		list_print(hdrscan);
+		printf("\n cached: ");
+		list_print(c->hdrscan);
+		printf("\n");
+
+		list_free(c->includes);
+		list_free(c->hdrscan);
+		c->includes = 0;
+		c->hdrscan = 0;
+	    } else {
+		if (DEBUG_HEADER)
+		    printf ("using header cache for %s\n", t->boundname);
+		c->age = 0;
+		++hits;
+		l = list_copy (0, c->includes);
+		return l;
+	    }
+	} else {
+	    if (DEBUG_HEADER)
+	        printf ("header cache out of date for %s\n", t->boundname);
+	    list_free (c->includes);
+	    list_free(c->hdrscan);
+	    c->includes = 0;
+	    c->hdrscan = 0;
+	}
+    } else {
+	if (hashenter (hcachehash, (HASHDATA **)&c)) {
+	    c->boundname = newstr (c->boundname);
+	    c->next = hcachelist;
+	    hcachelist = c;
+	}
+    }
+
+    /* 'c' points at the cache entry.  Its out of date. */
+
+    l = headers1 (0, t->boundname, rec, re);
+
+    c->time = t->time;
+    c->age = 0;
+    c->includes = list_copy (0, l);
+    c->hdrscan = list_copy(0, hdrscan);
+
+    return l;
+}
+
+#endif

src/tools/jam/hcache.h

@@ -0,0 +1,11 @@
+/*
+ * This file is not part of Jam
+ */
+
+/*
+ * hcache.h - handle #includes in source files
+ */
+
+void hcache_init(void);
+void hcache_done(void);
+LIST *hcache(TARGET *t, int rec, regexp *re[], LIST *hdrscan);

src/tools/jam/headers.c

@@ -14,6 +14,10 @@
 # include "headers.h"
 # include "newstr.h"
 
+#ifdef OPT_HEADER_CACHE_EXT
+# include "hcache.h"
+#endif
+
 /*
  * headers.c - handle #includes in source files
  *
@@ -35,7 +39,9 @@
  *		just to invoke a rule.
  */
 
+#ifndef OPT_HEADER_CACHE_EXT
 static LIST *headers1( LIST *l, char *file, int rec, regexp *re[] );
+#endif
 
 /*
  * headers() - scan a target for include files and call HDRRULE
@@ -73,7 +79,11 @@ headers( TARGET *t )
 
 	lol_init( &lol );
 	lol_add( &lol, list_new( L0, t->name ) );
+#ifdef OPT_HEADER_CACHE_EXT
+	lol_add( &lol, hcache( t, rec, re, var_get("HDRSCAN") ) );
+#else
 	lol_add( &lol, headers1( headlist, t->boundname, rec, re ) );
+#endif
 
 	if( lol_get( &lol, 1 ) )
 	    list_free( evaluate_rule( hdrrule->string, &lol, L0 ) );
@@ -90,7 +100,11 @@ headers( TARGET *t )
  * headers1() - using regexp, scan a file and build include LIST
  */
 
+#ifdef OPT_HEADER_CACHE_EXT
+LIST *
+#else
 static LIST *
+#endif
 headers1( 
 	LIST	*l,
 	char	*file,

src/tools/jam/headers.h

@@ -9,3 +9,8 @@
  */
 
 void headers( TARGET *t );
+
+#ifdef OPT_HEADER_CACHE_EXT
+struct regexp;
+LIST *headers1( LIST *l, char *file, int rec, struct regexp *re[] );
+#endif

src/tools/jam/jambase.c

@@ -759,6 +759,7 @@ char *jambase[] = {
 "LOCATE_SOURCE = $(ALL_LOCATE_TARGET) $(SUBDIR) ;\n",
 "LOCATE_TARGET = $(ALL_LOCATE_TARGET) $(SUBDIR) ;\n",
 "SOURCE_GRIST = [ FGrist $(<[2-]) ] ;\n",
+"HDRGRIST = $(SOURCE_GRIST) ;\n",
 "SUBDIRCCFLAGS = ;\n",
 "SUBDIRC++FLAGS = ;\n",
 "SUBDIRHDRS = ;\n",
@@ -817,23 +818,7 @@ char *jambase[] = {
 "}\n",
 "rule FGristSourceFiles\n",
 "{\n",
-"if ! $(SOURCE_GRIST)\n",
+"return [ FGristFiles $(<) ] ;\n",
-"{\n",
-"return $(<) ;\n",
-"}\n",
-"else \n",
-"{\n",
-"local _i _o ;\n",
-"for _i in $(<)\n",
-"{\n",
-"switch $(_i)\n",
-"{\n",
-"case *.h :	_o += $(_i) ;\n",
-"case * : 	_o += $(_i:G=$(SOURCE_GRIST)) ;\n",
-"}\n",
-"}\n",
-"return $(_o) ;\n",
-"}\n",
 "}\n",
 "rule FIsPrefix\n",
 "{\n",

src/tools/jam/make.c

@@ -110,6 +110,10 @@ make(
 	COUNTS counts[1];
 	int status = 0;		/* 1 if anything fails */
 
+#ifdef OPT_HEADER_CACHE_EXT
+	hcache_init();
+#endif
+
 	memset( (char *)counts, 0, sizeof( *counts ) );
 
 	for( i = 0; i < n_targets; i++ )
@@ -133,6 +137,10 @@ make(
 		printf( "...can't make %d target(s)...\n", counts->cantmake );
 	}
 
+#ifdef OPT_HEADER_CACHE_EXT
+	hcache_done();
+#endif
+
 	status = counts->cantfind || counts->cantmake;
 
 	for( i = 0; i < n_targets; i++ )