Commit 86e9749d authored by Matt Holt's avatar Matt Holt

Merge pull request #204 from abiosoft/master

markdown: Add .Links action and flattened metadata structure

.Links available only for generated sites and variables no longer in a [variables] category in metadata (flat structure).
parents a585379b aa89f30f
......@@ -34,6 +34,10 @@ func Markdown(c *Controller) (middleware.Middleware, error) {
continue
}
if err := markdown.GenerateLinks(md, &cfg); err != nil {
return err
}
// If generated site already exists, clear it out
_, err := os.Stat(cfg.StaticDir)
if err == nil {
......
......@@ -4,9 +4,11 @@ package markdown
import (
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"sync"
"github.com/mholt/caddy/middleware"
"github.com/russross/blackfriday"
......@@ -64,13 +66,19 @@ type Config struct {
// Map of request URL to static files generated
StaticFiles map[string]string
// Links to all markdown pages ordered by date.
Links []PageLink
// Directory to store static files
StaticDir string
sync.RWMutex
}
// ServeHTTP implements the http.Handler interface.
func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
for _, m := range md.Configs {
for i := range md.Configs {
m := &md.Configs[i]
if !middleware.Path(r.URL.Path).Matches(m.PathScope) {
continue
}
......@@ -114,6 +122,13 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
}
}
if m.StaticDir != "" {
// Markdown modified or new. Update links.
if err := GenerateLinks(md, m); err != nil {
log.Println(err)
}
}
body, err := ioutil.ReadAll(f)
if err != nil {
return http.StatusInternalServerError, err
......@@ -124,7 +139,7 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
Req: r,
URL: r.URL,
}
html, err := md.Process(m, fpath, body, ctx)
html, err := md.Process(*m, fpath, body, ctx)
if err != nil {
return http.StatusInternalServerError, err
}
......
......@@ -4,6 +4,7 @@ import (
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"github.com/mholt/caddy/middleware"
......@@ -18,20 +19,24 @@ func TestMarkdown(t *testing.T) {
FileSys: http.Dir("./testdata"),
Configs: []Config{
Config{
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/blog",
Extensions: []string{".md"},
Styles: []string{},
Scripts: []string{},
Templates: templates,
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/blog",
Extensions: []string{".md"},
Styles: []string{},
Scripts: []string{},
Templates: templates,
StaticDir: DefaultStaticDir,
StaticFiles: make(map[string]string),
},
Config{
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/log",
Extensions: []string{".md"},
Styles: []string{"/resources/css/log.css", "/resources/css/default.css"},
Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"},
Templates: make(map[string]string),
Renderer: blackfriday.HtmlRenderer(0, "", ""),
PathScope: "/log",
Extensions: []string{".md"},
Styles: []string{"/resources/css/log.css", "/resources/css/default.css"},
Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"},
Templates: make(map[string]string),
StaticDir: DefaultStaticDir,
StaticFiles: make(map[string]string),
},
},
IndexFiles: []string{"index.html"},
......@@ -123,4 +128,34 @@ func getTrue() bool {
if respBody != expectedBody {
t.Fatalf("Expected body: %v got: %v", expectedBody, respBody)
}
expectedLinks := []string{
"/blog/test.md",
"/log/test.md",
}
for i, c := range md.Configs {
if c.Links[0].Url != expectedLinks[i] {
t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].Url)
}
}
// attempt to trigger race condition
var w sync.WaitGroup
f := func() {
req, err := http.NewRequest("GET", "/log/test.md", nil)
if err != nil {
t.Fatalf("Could not create HTTP request: %v", err)
}
rec := httptest.NewRecorder()
md.ServeHTTP(rec, req)
w.Done()
}
for i := 0; i < 5; i++ {
w.Add(1)
go f()
}
w.Wait()
}
......@@ -9,14 +9,7 @@ import (
"github.com/BurntSushi/toml"
"gopkg.in/yaml.v2"
)
var (
parsers = []MetadataParser{
&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
}
"time"
)
// Metadata stores a page's metadata
......@@ -27,20 +20,31 @@ type Metadata struct {
// Page template
Template string
// Publish date
Date time.Time
// Variables to be used with Template
Variables map[string]string
}
// load loads parsed values in parsedMap into Metadata
func (m *Metadata) load(parsedMap map[string]interface{}) {
if template, ok := parsedMap["title"]; ok {
m.Title, _ = template.(string)
if title, ok := parsedMap["title"]; ok {
m.Title, _ = title.(string)
}
if template, ok := parsedMap["template"]; ok {
m.Template, _ = template.(string)
}
if variables, ok := parsedMap["variables"]; ok {
m.Variables, _ = variables.(map[string]string)
if date, ok := parsedMap["date"].(string); ok {
if t, err := time.Parse(timeLayout, date); err == nil {
m.Date = t
}
}
// store everything as a variable
for key, val := range parsedMap {
if v, ok := val.(string); ok {
m.Variables[key] = v
}
}
}
......@@ -62,7 +66,7 @@ type MetadataParser interface {
Metadata() Metadata
}
// JSONMetadataParser is the MetdataParser for JSON
// JSONMetadataParser is the MetadataParser for JSON
type JSONMetadataParser struct {
metadata Metadata
}
......@@ -76,16 +80,6 @@ func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) {
if err := decoder.Decode(&m); err != nil {
return b, err
}
if vars, ok := m["variables"].(map[string]interface{}); ok {
vars1 := make(map[string]string)
for k, v := range vars {
if val, ok := v.(string); ok {
vars1[k] = val
}
}
m["variables"] = vars1
}
j.metadata.load(m)
// Retrieve remaining bytes after decoding
......@@ -129,15 +123,6 @@ func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) {
if err := toml.Unmarshal(b, &m); err != nil {
return markdown, err
}
if vars, ok := m["variables"].(map[string]interface{}); ok {
vars1 := make(map[string]string)
for k, v := range vars {
if val, ok := v.(string); ok {
vars1[k] = val
}
}
m["variables"] = vars1
}
t.metadata.load(m)
return markdown, nil
}
......@@ -174,21 +159,6 @@ func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) {
if err := yaml.Unmarshal(b, &m); err != nil {
return markdown, err
}
// convert variables (if present) to map[string]interface{}
// to match expected type
if vars, ok := m["variables"].(map[interface{}]interface{}); ok {
vars1 := make(map[string]string)
for k, v := range vars {
if key, ok := k.(string); ok {
if val, ok := v.(string); ok {
vars1[key] = val
}
}
}
m["variables"] = vars1
}
y.metadata.load(m)
return markdown, nil
}
......@@ -260,10 +230,19 @@ func findParser(b []byte) MetadataParser {
return nil
}
line = bytes.TrimSpace(line)
for _, parser := range parsers {
for _, parser := range parsers() {
if bytes.Equal(parser.Opening(), line) {
return parser
}
}
return nil
}
// parsers returns all available parsers
func parsers() []MetadataParser {
return []MetadataParser{
&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}},
}
}
......@@ -11,13 +11,11 @@ import (
var TOML = [4]string{`
title = "A title"
template = "default"
[variables]
name = "value"
`,
`+++
title = "A title"
template = "default"
[variables]
name = "value"
+++
Page content
......@@ -25,7 +23,6 @@ Page content
`+++
title = "A title"
template = "default"
[variables]
name = "value"
`,
`title = "A title" template = "default" [variables] name = "value"`,
......@@ -34,38 +31,31 @@ name = "value"
var YAML = [4]string{`
title : A title
template : default
variables :
name : value
name : value
`,
`---
title : A title
template : default
variables :
name : value
name : value
---
Page content
`,
`---
title : A title
template : default
variables :
name : value
name : value
`,
`title : A title template : default variables : name : value`,
}
var JSON = [4]string{`
"title" : "A title",
"template" : "default",
"variables" : {
"name" : "value"
}
"name" : "value"
`,
`{
"title" : "A title",
"template" : "default",
"variables" : {
"name" : "value"
}
"name" : "value"
}
Page content
`,
......@@ -73,17 +63,13 @@ Page content
{
"title" : "A title",
"template" : "default",
"variables" : {
"name" : "value"
}
"name" : "value"
`,
`
{{
"title" : "A title",
"template" : "default",
"variables" : {
"name" : "value"
}
"name" : "value"
}
`,
}
......@@ -96,9 +82,13 @@ func check(t *testing.T, err error) {
func TestParsers(t *testing.T) {
expected := Metadata{
Title: "A title",
Template: "default",
Variables: map[string]string{"name": "value"},
Title: "A title",
Template: "default",
Variables: map[string]string{
"name": "value",
"title": "A title",
"template": "default",
},
}
compare := func(m Metadata) bool {
if m.Title != expected.Title {
......@@ -112,7 +102,7 @@ func TestParsers(t *testing.T) {
return false
}
}
return len(m.Variables) == 1
return len(m.Variables) == len(expected.Variables)
}
data := []struct {
......@@ -120,9 +110,9 @@ func TestParsers(t *testing.T) {
testData [4]string
name string
}{
{&JSONMetadataParser{}, JSON, "json"},
{&YAMLMetadataParser{}, YAML, "yaml"},
{&TOMLMetadataParser{}, TOML, "toml"},
{&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"},
{&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"},
{&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"},
}
for _, v := range data {
......
package markdown
import (
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/russross/blackfriday"
)
const (
// Date format YYYY-MM-DD HH:MM:SS
timeLayout = `2006-01-02 15:04:05`
// Length of page summary.
summaryLen = 150
)
// PageLink represents a statically generated markdown page.
type PageLink struct {
Title string
Summary string
Date time.Time
Url string
}
// byDate sorts PageLink by newest date to oldest.
type byDate []PageLink
func (p byDate) Len() int { return len(p) }
func (p byDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p byDate) Less(i, j int) bool { return p[i].Date.After(p[j].Date) }
type linkGen struct {
generating bool
waiters int
lastErr error
sync.RWMutex
sync.WaitGroup
}
func (l *linkGen) addWaiter() {
l.WaitGroup.Add(1)
l.waiters++
}
func (l *linkGen) discardWaiters() {
l.Lock()
defer l.Unlock()
for i := 0; i < l.waiters; i++ {
l.Done()
}
}
func (l *linkGen) started() bool {
l.RLock()
defer l.RUnlock()
return l.generating
}
func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
l.Lock()
l.generating = true
l.Unlock()
fp := filepath.Join(md.Root, cfg.PathScope)
cfg.Links = []PageLink{}
cfg.Lock()
l.lastErr = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
for _, ext := range cfg.Extensions {
if !info.IsDir() && strings.HasSuffix(info.Name(), ext) {
// Load the file
body, err := ioutil.ReadFile(path)
if err != nil {
return err
}
// Get the relative path as if it were a HTTP request,
// then prepend with "/" (like a real HTTP request)
reqPath, err := filepath.Rel(md.Root, path)
if err != nil {
return err
}
reqPath = "/" + reqPath
parser := findParser(body)
if parser == nil {
// no metadata, ignore.
continue
}
summary, err := parser.Parse(body)
if err != nil {
return err
}
if len(summary) > summaryLen {
summary = summary[:summaryLen]
}
metadata := parser.Metadata()
cfg.Links = append(cfg.Links, PageLink{
Title: metadata.Title,
Url: reqPath,
Date: metadata.Date,
Summary: string(blackfriday.Markdown(summary, PlaintextRenderer{}, 0)),
})
break // don't try other file extensions
}
}
return nil
})
// sort by newest date
sort.Sort(byDate(cfg.Links))
cfg.Unlock()
l.Lock()
l.generating = false
l.Unlock()
}
type linkGenerator struct {
gens map[*Config]*linkGen
sync.Mutex
}
var generator = linkGenerator{gens: make(map[*Config]*linkGen)}
// GenerateLinks generates links to all markdown files ordered by newest date.
// This blocks until link generation is done. When called by multiple goroutines,
// the first caller starts the generation and others only wait.
func GenerateLinks(md Markdown, cfg *Config) error {
generator.Lock()
// if link generator exists for config and running, wait.
if g, ok := generator.gens[cfg]; ok {
if g.started() {
g.addWaiter()
generator.Unlock()
g.Wait()
return g.lastErr
}
}
g := &linkGen{}
generator.gens[cfg] = g
generator.Unlock()
g.generateLinks(md, cfg)
g.discardWaiters()
return g.lastErr
}
......@@ -20,7 +20,8 @@ const (
type MarkdownData struct {
middleware.Context
Doc map[string]string
Doc map[string]string
Links []PageLink
}
// Process processes the contents of a page in b. It parses the metadata
......@@ -97,9 +98,14 @@ func (md Markdown) processTemplate(c Config, requestPath string, tmpl []byte, me
mdData := MarkdownData{
Context: ctx,
Doc: metadata.Variables,
Links: c.Links,
}
if err = t.Execute(b, mdData); err != nil {
c.RLock()
err = t.Execute(b, mdData)
c.RUnlock()
if err != nil {
return nil, err
}
......
package markdown
import (
"bytes"
)
type PlaintextRenderer struct{}
// Block-level callbacks
func (r PlaintextRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {}
func (r PlaintextRenderer) BlockQuote(out *bytes.Buffer, text []byte) {}
func (r PlaintextRenderer) BlockHtml(out *bytes.Buffer, text []byte) {}
func (r PlaintextRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) {}
func (r PlaintextRenderer) HRule(out *bytes.Buffer) {}
func (r PlaintextRenderer) List(out *bytes.Buffer, text func() bool, flags int) {}
func (r PlaintextRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {}
func (r PlaintextRenderer) Paragraph(out *bytes.Buffer, text func() bool) {
marker := out.Len()
if !text() {
out.Truncate(marker)
}
out.Write([]byte{' '})
}
func (r PlaintextRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {}
func (r PlaintextRenderer) TableRow(out *bytes.Buffer, text []byte) {}
func (r PlaintextRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {}
func (r PlaintextRenderer) TableCell(out *bytes.Buffer, text []byte, flags int) {}
func (r PlaintextRenderer) Footnotes(out *bytes.Buffer, text func() bool) {}
func (r PlaintextRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {}
func (r PlaintextRenderer) TitleBlock(out *bytes.Buffer, text []byte) {}
// Span-level callbacks
func (r PlaintextRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {}
func (r PlaintextRenderer) CodeSpan(out *bytes.Buffer, text []byte) {}
func (r PlaintextRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) {
out.Write(text)
}
func (r PlaintextRenderer) Emphasis(out *bytes.Buffer, text []byte) {
out.Write(text)
}
func (r PlaintextRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {}
func (r PlaintextRenderer) LineBreak(out *bytes.Buffer) {}
func (r PlaintextRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
out.Write(content)
}
func (r PlaintextRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) {}
func (r PlaintextRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) {
out.Write(text)
}
func (r PlaintextRenderer) StrikeThrough(out *bytes.Buffer, text []byte) {}
func (r PlaintextRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {}
// Low-level callbacks
func (r PlaintextRenderer) Entity(out *bytes.Buffer, entity []byte) {
out.Write(entity)
}
func (r PlaintextRenderer) NormalText(out *bytes.Buffer, text []byte) {
out.Write(text)
}
// Header and footer
func (r PlaintextRenderer) DocumentHeader(out *bytes.Buffer) {}
func (r PlaintextRenderer) DocumentFooter(out *bytes.Buffer) {}
func (r PlaintextRenderer) GetFlags() int { return 0 }
---
title: Markdown test
variables:
sitename: A Caddy website
sitename: A Caddy website
---
## Welcome on the blog
......
---
title: Markdown test
variables:
sitename: A Caddy website
sitename: A Caddy website
---
## Welcome on the blog
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment