Merge pull request #204 from abiosoft/master

markdown: Add .Links action and flattened metadata structure

.Links available only for generated sites and variables no longer in a [variables] category in metadata (flat structure).

config/setup/markdown.go

@@ -34,6 +34,10 @@ func Markdown(c *Controller) (middleware.Middleware, error) {
 				continue
 			}
 
+			if err := markdown.GenerateLinks(md, &cfg); err != nil {
+				return err
+			}
+
 			// If generated site already exists, clear it out
 			_, err := os.Stat(cfg.StaticDir)
 			if err == nil {

middleware/markdown/markdown.go

@@ -4,9 +4,11 @@ package markdown
 
 import (
 	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/mholt/caddy/middleware"
 	"github.com/russross/blackfriday"
@@ -64,13 +66,19 @@ type Config struct {
 	// Map of request URL to static files generated
 	StaticFiles map[string]string
 
+	// Links to all markdown pages ordered by date.
+	Links []PageLink
+
 	// Directory to store static files
 	StaticDir string
+
+	sync.RWMutex
 }
 
 // ServeHTTP implements the http.Handler interface.
 func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
-	for _, m := range md.Configs {
+	for i := range md.Configs {
+		m := &md.Configs[i]
 		if !middleware.Path(r.URL.Path).Matches(m.PathScope) {
 			continue
 		}
@@ -114,6 +122,13 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 					}
 				}
 
+				if m.StaticDir != "" {
+					// Markdown modified or new. Update links.
+					if err := GenerateLinks(md, m); err != nil {
+						log.Println(err)
+					}
+				}
+
 				body, err := ioutil.ReadAll(f)
 				if err != nil {
 					return http.StatusInternalServerError, err
@@ -124,7 +139,7 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 					Req:  r,
 					URL:  r.URL,
 				}
-				html, err := md.Process(m, fpath, body, ctx)
+				html, err := md.Process(*m, fpath, body, ctx)
 				if err != nil {
 					return http.StatusInternalServerError, err
 				}

middleware/markdown/markdown_test.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/mholt/caddy/middleware"
@@ -18,20 +19,24 @@ func TestMarkdown(t *testing.T) {
 		FileSys: http.Dir("./testdata"),
 		Configs: []Config{
 			Config{
-				Renderer:   blackfriday.HtmlRenderer(0, "", ""),
-				PathScope:  "/blog",
-				Extensions: []string{".md"},
-				Styles:     []string{},
-				Scripts:    []string{},
-				Templates:  templates,
+				Renderer:    blackfriday.HtmlRenderer(0, "", ""),
+				PathScope:   "/blog",
+				Extensions:  []string{".md"},
+				Styles:      []string{},
+				Scripts:     []string{},
+				Templates:   templates,
+				StaticDir:   DefaultStaticDir,
+				StaticFiles: make(map[string]string),
 			},
 			Config{
-				Renderer:   blackfriday.HtmlRenderer(0, "", ""),
-				PathScope:  "/log",
-				Extensions: []string{".md"},
-				Styles:     []string{"/resources/css/log.css", "/resources/css/default.css"},
-				Scripts:    []string{"/resources/js/log.js", "/resources/js/default.js"},
-				Templates:  make(map[string]string),
+				Renderer:    blackfriday.HtmlRenderer(0, "", ""),
+				PathScope:   "/log",
+				Extensions:  []string{".md"},
+				Styles:      []string{"/resources/css/log.css", "/resources/css/default.css"},
+				Scripts:     []string{"/resources/js/log.js", "/resources/js/default.js"},
+				Templates:   make(map[string]string),
+				StaticDir:   DefaultStaticDir,
+				StaticFiles: make(map[string]string),
 			},
 		},
 		IndexFiles: []string{"index.html"},
@@ -123,4 +128,34 @@ func getTrue() bool {
 	if respBody != expectedBody {
 		t.Fatalf("Expected body: %v got: %v", expectedBody, respBody)
 	}
+
+	expectedLinks := []string{
+		"/blog/test.md",
+		"/log/test.md",
+	}
+
+	for i, c := range md.Configs {
+		if c.Links[0].Url != expectedLinks[i] {
+			t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].Url)
+		}
+	}
+
+	// attempt to trigger race condition
+	var w sync.WaitGroup
+	f := func() {
+		req, err := http.NewRequest("GET", "/log/test.md", nil)
+		if err != nil {
+			t.Fatalf("Could not create HTTP request: %v", err)
+		}
+		rec := httptest.NewRecorder()
+
+		md.ServeHTTP(rec, req)
+		w.Done()
+	}
+	for i := 0; i < 5; i++ {
+		w.Add(1)
+		go f()
+	}
+	w.Wait()
+
 }

middleware/markdown/metadata.go

@@ -9,14 +9,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"gopkg.in/yaml.v2"
-)
-
-var (
-	parsers = []MetadataParser{
-		&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
-		&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
-		&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
-	}
+	"time"
 )
 
 // Metadata stores a page's metadata
@@ -27,20 +20,31 @@ type Metadata struct {
 	// Page template
 	Template string
 
+	// Publish date
+	Date time.Time
+
 	// Variables to be used with Template
 	Variables map[string]string
 }
 
 // load loads parsed values in parsedMap into Metadata
 func (m *Metadata) load(parsedMap map[string]interface{}) {
-	if template, ok := parsedMap["title"]; ok {
-		m.Title, _ = template.(string)
+	if title, ok := parsedMap["title"]; ok {
+		m.Title, _ = title.(string)
 	}
 	if template, ok := parsedMap["template"]; ok {
 		m.Template, _ = template.(string)
 	}
-	if variables, ok := parsedMap["variables"]; ok {
-		m.Variables, _ = variables.(map[string]string)
+	if date, ok := parsedMap["date"].(string); ok {
+		if t, err := time.Parse(timeLayout, date); err == nil {
+			m.Date = t
+		}
+	}
+	// store everything as a variable
+	for key, val := range parsedMap {
+		if v, ok := val.(string); ok {
+			m.Variables[key] = v
+		}
 	}
 }
 
@@ -62,7 +66,7 @@ type MetadataParser interface {
 	Metadata() Metadata
 }
 
-// JSONMetadataParser is the MetdataParser for JSON
+// JSONMetadataParser is the MetadataParser for JSON
 type JSONMetadataParser struct {
 	metadata Metadata
 }
@@ -76,16 +80,6 @@ func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) {
 	if err := decoder.Decode(&m); err != nil {
 		return b, err
 	}
-	if vars, ok := m["variables"].(map[string]interface{}); ok {
-		vars1 := make(map[string]string)
-		for k, v := range vars {
-			if val, ok := v.(string); ok {
-				vars1[k] = val
-			}
-		}
-		m["variables"] = vars1
-	}
-
 	j.metadata.load(m)
 
 	// Retrieve remaining bytes after decoding
@@ -129,15 +123,6 @@ func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) {
 	if err := toml.Unmarshal(b, &m); err != nil {
 		return markdown, err
 	}
-	if vars, ok := m["variables"].(map[string]interface{}); ok {
-		vars1 := make(map[string]string)
-		for k, v := range vars {
-			if val, ok := v.(string); ok {
-				vars1[k] = val
-			}
-		}
-		m["variables"] = vars1
-	}
 	t.metadata.load(m)
 	return markdown, nil
 }
@@ -174,21 +159,6 @@ func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) {
 	if err := yaml.Unmarshal(b, &m); err != nil {
 		return markdown, err
 	}
-
-	// convert variables (if present) to map[string]interface{}
-	// to match expected type
-	if vars, ok := m["variables"].(map[interface{}]interface{}); ok {
-		vars1 := make(map[string]string)
-		for k, v := range vars {
-			if key, ok := k.(string); ok {
-				if val, ok := v.(string); ok {
-					vars1[key] = val
-				}
-			}
-		}
-		m["variables"] = vars1
-	}
-
 	y.metadata.load(m)
 	return markdown, nil
 }
@@ -260,10 +230,19 @@ func findParser(b []byte) MetadataParser {
 		return nil
 	}
 	line = bytes.TrimSpace(line)
-	for _, parser := range parsers {
+	for _, parser := range parsers() {
 		if bytes.Equal(parser.Opening(), line) {
 			return parser
 		}
 	}
 	return nil
 }
+
+// parsers returns all available parsers
+func parsers() []MetadataParser {
+	return []MetadataParser{
+		&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
+		&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
+		&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
+	}
+}

middleware/markdown/metadata_test.go

@@ -11,13 +11,11 @@ import (
 var TOML = [4]string{`
 title = "A title"
 template = "default"
-[variables]
 name = "value"
 `,
 	`+++
 title = "A title"
 template = "default"
-[variables]
 name = "value"
 +++
 Page content
@@ -25,7 +23,6 @@ Page content
 	`+++
 title = "A title"
 template = "default"
-[variables]
 name = "value"
 	`,
 	`title = "A title" template = "default" [variables] name = "value"`,
@@ -34,38 +31,31 @@ name = "value"
 var YAML = [4]string{`
 title : A title
 template : default
-variables :
-  name : value
+name : value
 `,
 	`---
 title : A title
 template : default
-variables :
-  name : value
+name : value
 ---
 Page content
 `,
 	`---
 title : A title
 template : default
-variables :
-  name : value
+name : value
 `,
 	`title : A title template : default variables : name : value`,
 }
 var JSON = [4]string{`
 	"title" : "A title",
 	"template" : "default",
-	"variables" : {
-		"name" : "value"
-	}
+	"name" : "value"
 `,
 	`{
 	"title" : "A title",
 	"template" : "default",
-	"variables" : {
-		"name" : "value"
-	}
+	"name" : "value"
 }
 Page content
 `,
@@ -73,17 +63,13 @@ Page content
 {
 	"title" : "A title",
 	"template" : "default",
-	"variables" : {
-		"name" : "value"
-	}
+	"name" : "value"
 `,
 	`
 {{
 	"title" : "A title",
 	"template" : "default",
-	"variables" : {
-		"name" : "value"
-	}
+	"name" : "value"
 }
 	`,
 }
@@ -96,9 +82,13 @@ func check(t *testing.T, err error) {
 
 func TestParsers(t *testing.T) {
 	expected := Metadata{
-		Title:     "A title",
-		Template:  "default",
-		Variables: map[string]string{"name": "value"},
+		Title:    "A title",
+		Template: "default",
+		Variables: map[string]string{
+			"name":     "value",
+			"title":    "A title",
+			"template": "default",
+		},
 	}
 	compare := func(m Metadata) bool {
 		if m.Title != expected.Title {
@@ -112,7 +102,7 @@ func TestParsers(t *testing.T) {
 				return false
 			}
 		}
-		return len(m.Variables) == 1
+		return len(m.Variables) == len(expected.Variables)
 	}
 
 	data := []struct {
@@ -120,9 +110,9 @@ func TestParsers(t *testing.T) {
 		testData [4]string
 		name     string
 	}{
-		{&JSONMetadataParser{}, JSON, "json"},
-		{&YAMLMetadataParser{}, YAML, "yaml"},
-		{&TOMLMetadataParser{}, TOML, "toml"},
+		{&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"},
+		{&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"},
+		{&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"},
 	}
 
 	for _, v := range data {

middleware/markdown/page.go

+package markdown
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/russross/blackfriday"
+)
+
+const (
+	// Date format YYYY-MM-DD HH:MM:SS
+	timeLayout = `2006-01-02 15:04:05`
+
+	// Length of page summary.
+	summaryLen = 150
+)
+
+// PageLink represents a statically generated markdown page.
+type PageLink struct {
+	Title   string
+	Summary string
+	Date    time.Time
+	Url     string
+}
+
+// byDate sorts PageLink by newest date to oldest.
+type byDate []PageLink
+
+func (p byDate) Len() int           { return len(p) }
+func (p byDate) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p byDate) Less(i, j int) bool { return p[i].Date.After(p[j].Date) }
+
+type linkGen struct {
+	generating bool
+	waiters    int
+	lastErr    error
+	sync.RWMutex
+	sync.WaitGroup
+}
+
+func (l *linkGen) addWaiter() {
+	l.WaitGroup.Add(1)
+	l.waiters++
+}
+
+func (l *linkGen) discardWaiters() {
+	l.Lock()
+	defer l.Unlock()
+	for i := 0; i < l.waiters; i++ {
+		l.Done()
+	}
+}
+
+func (l *linkGen) started() bool {
+	l.RLock()
+	defer l.RUnlock()
+	return l.generating
+}
+
+func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
+	l.Lock()
+	l.generating = true
+	l.Unlock()
+
+	fp := filepath.Join(md.Root, cfg.PathScope)
+
+	cfg.Links = []PageLink{}
+
+	cfg.Lock()
+	l.lastErr = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
+		for _, ext := range cfg.Extensions {
+			if !info.IsDir() && strings.HasSuffix(info.Name(), ext) {
+				// Load the file
+				body, err := ioutil.ReadFile(path)
+				if err != nil {
+					return err
+				}
+
+				// Get the relative path as if it were a HTTP request,
+				// then prepend with "/" (like a real HTTP request)
+				reqPath, err := filepath.Rel(md.Root, path)
+				if err != nil {
+					return err
+				}
+				reqPath = "/" + reqPath
+
+				parser := findParser(body)
+				if parser == nil {
+					// no metadata, ignore.
+					continue
+				}
+				summary, err := parser.Parse(body)
+				if err != nil {
+					return err
+				}
+
+				if len(summary) > summaryLen {
+					summary = summary[:summaryLen]
+				}
+
+				metadata := parser.Metadata()
+
+				cfg.Links = append(cfg.Links, PageLink{
+					Title:   metadata.Title,
+					Url:     reqPath,
+					Date:    metadata.Date,
+					Summary: string(blackfriday.Markdown(summary, PlaintextRenderer{}, 0)),
+				})
+
+				break // don't try other file extensions
+			}
+		}
+
+		return nil
+	})
+
+	// sort by newest date
+	sort.Sort(byDate(cfg.Links))
+	cfg.Unlock()
+
+	l.Lock()
+	l.generating = false
+	l.Unlock()
+}
+
+type linkGenerator struct {
+	gens map[*Config]*linkGen
+	sync.Mutex
+}
+
+var generator = linkGenerator{gens: make(map[*Config]*linkGen)}
+
+// GenerateLinks generates links to all markdown files ordered by newest date.
+// This blocks until link generation is done. When called by multiple goroutines,
+// the first caller starts the generation and others only wait.
+func GenerateLinks(md Markdown, cfg *Config) error {
+	generator.Lock()
+
+	// if link generator exists for config and running, wait.
+	if g, ok := generator.gens[cfg]; ok {
+		if g.started() {
+			g.addWaiter()
+			generator.Unlock()
+			g.Wait()
+			return g.lastErr
+		}
+	}
+
+	g := &linkGen{}
+	generator.gens[cfg] = g
+	generator.Unlock()
+
+	g.generateLinks(md, cfg)
+	g.discardWaiters()
+	return g.lastErr
+}

middleware/markdown/process.go

@@ -20,7 +20,8 @@ const (
 
 type MarkdownData struct {
 	middleware.Context
-	Doc map[string]string
+	Doc   map[string]string
+	Links []PageLink
 }
 
 // Process processes the contents of a page in b. It parses the metadata
@@ -97,9 +98,14 @@ func (md Markdown) processTemplate(c Config, requestPath string, tmpl []byte, me
 	mdData := MarkdownData{
 		Context: ctx,
 		Doc:     metadata.Variables,
+		Links:   c.Links,
 	}
 
-	if err = t.Execute(b, mdData); err != nil {
+	c.RLock()
+	err = t.Execute(b, mdData)
+	c.RUnlock()
+
+	if err != nil {
 		return nil, err
 	}
 

middleware/markdown/renderer.go

+package markdown
+
+import (
+	"bytes"
+)
+
+type PlaintextRenderer struct{}
+
+// Block-level callbacks
+
+func (r PlaintextRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {}
+
+func (r PlaintextRenderer) BlockQuote(out *bytes.Buffer, text []byte) {}
+
+func (r PlaintextRenderer) BlockHtml(out *bytes.Buffer, text []byte) {}
+
+func (r PlaintextRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) {}
+
+func (r PlaintextRenderer) HRule(out *bytes.Buffer) {}
+
+func (r PlaintextRenderer) List(out *bytes.Buffer, text func() bool, flags int) {}
+
+func (r PlaintextRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {}
+
+func (r PlaintextRenderer) Paragraph(out *bytes.Buffer, text func() bool) {
+	marker := out.Len()
+	if !text() {
+		out.Truncate(marker)
+	}
+	out.Write([]byte{' '})
+}
+
+func (r PlaintextRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {}
+
+func (r PlaintextRenderer) TableRow(out *bytes.Buffer, text []byte) {}
+
+func (r PlaintextRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {}
+
+func (r PlaintextRenderer) TableCell(out *bytes.Buffer, text []byte, flags int) {}
+
+func (r PlaintextRenderer) Footnotes(out *bytes.Buffer, text func() bool) {}
+
+func (r PlaintextRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {}
+
+func (r PlaintextRenderer) TitleBlock(out *bytes.Buffer, text []byte) {}
+
+// Span-level callbacks
+
+func (r PlaintextRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {}
+
+func (r PlaintextRenderer) CodeSpan(out *bytes.Buffer, text []byte) {}
+
+func (r PlaintextRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) {
+	out.Write(text)
+}
+
+func (r PlaintextRenderer) Emphasis(out *bytes.Buffer, text []byte) {
+	out.Write(text)
+}
+
+func (r PlaintextRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {}
+
+func (r PlaintextRenderer) LineBreak(out *bytes.Buffer) {}
+
+func (r PlaintextRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
+	out.Write(content)
+}
+func (r PlaintextRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) {}
+
+func (r PlaintextRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) {
+	out.Write(text)
+}
+func (r PlaintextRenderer) StrikeThrough(out *bytes.Buffer, text []byte) {}
+
+func (r PlaintextRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {}
+
+// Low-level callbacks
+
+func (r PlaintextRenderer) Entity(out *bytes.Buffer, entity []byte) {
+	out.Write(entity)
+}
+
+func (r PlaintextRenderer) NormalText(out *bytes.Buffer, text []byte) {
+	out.Write(text)
+}
+
+// Header and footer
+
+func (r PlaintextRenderer) DocumentHeader(out *bytes.Buffer) {}
+
+func (r PlaintextRenderer) DocumentFooter(out *bytes.Buffer) {}
+
+func (r PlaintextRenderer) GetFlags() int { return 0 }

middleware/markdown/testdata/blog/test.md

 ---
 title: Markdown test
-variables:
-    sitename: A Caddy website
+sitename: A Caddy website
 ---
 
 ## Welcome on the blog

middleware/markdown/testdata/log/test.md

 ---
 title: Markdown test
-variables:
-    sitename: A Caddy website
+sitename: A Caddy website
 ---
 
 ## Welcome on the blog