diff options
| author | Shulhan <ms@kilabit.info> | 2023-05-14 17:06:21 +0700 |
|---|---|---|
| committer | Shulhan <ms@kilabit.info> | 2023-05-14 17:13:56 +0700 |
| commit | fad7cd21134a4cac75f876bce866830c35042f8e (patch) | |
| tree | dd55dd5c6edf2b5bfb8f0bf7d6e1cb12b15712eb | |
| parent | c109a3dd026cf6fbdc147d5a2c17535569d8964d (diff) | |
| download | ciigo-fad7cd21134a4cac75f876bce866830c35042f8e.tar.xz | |
all: bring back support for Markdown
I use two remote repositories: GitHub and SourceHut.
GitHub support rendering README using asciidoc while SourceHut not.
This cause the repository that use README.adoc rendered as text in
SourceHut which make the repository page less readable.
Also, the pkg.go.dev now render README but only support Markdown.
Since we cannot control the SourceHut and go.dev, the only option is
to support converting Markdown in ciigo so I can write README using
Markdown and the rest of documentation using Asciidoc.
| -rw-r--r-- | README | 1 | ||||
| -rw-r--r-- | ciigo.go | 23 | ||||
| -rw-r--r-- | ciigo_test.go | 5 | ||||
| -rw-r--r-- | cmd/ciigo-example/static.go | 1 | ||||
| -rw-r--r-- | converter.go | 91 | ||||
| -rw-r--r-- | file_markup.go (renamed from filemarkup.go) | 36 | ||||
| -rw-r--r-- | filehtml.go | 36 | ||||
| -rw-r--r-- | go.mod | 3 | ||||
| -rw-r--r-- | go.sum | 8 | ||||
| -rw-r--r-- | testdata/ex/clu/de/markdown.md | 1 | ||||
| -rw-r--r-- | testdata/in/clu/de/markdown.md | 1 | ||||
| -rw-r--r-- | testdata/watcher_test.txt | 29 | ||||
| -rw-r--r-- | watcher.go | 11 | ||||
| -rw-r--r-- | watcher_test.go | 114 |
14 files changed, 315 insertions, 45 deletions
@@ -8,6 +8,7 @@ `ciigo` is a library and a program to write static web server with embedded files using https://asciidoctor.org/docs/what-is-asciidoc/[AsciiDoc^] +https://www.markdownguide.org/[Markdown^] markup format. @@ -22,6 +22,7 @@ import ( const ( extAsciidoc = `.adoc` + extMarkdown = `.md` internalTemplatePath = `_internal/.template` ) @@ -30,6 +31,7 @@ var ( defExcludes = []string{ `.*\.adoc$`, + `.*\.md$`, `^\..*`, } ) @@ -43,7 +45,7 @@ func Convert(opts *ConvertOptions) (err error) { logp = `Convert` converter *Converter - fileMarkups map[string]*fileMarkup + fileMarkups map[string]*FileMarkup ) if opts == nil { @@ -84,7 +86,7 @@ func GoEmbed(opts *EmbedOptions) (err error) { logp = `GoEmbed` converter *Converter - fileMarkups map[string]*fileMarkup + fileMarkups map[string]*FileMarkup mfs *memfs.MemFS mfsOpts *memfs.Options convertForce bool @@ -245,27 +247,32 @@ func isHtmlTemplateNewer(opts *EmbedOptions) bool { return fiHtmlTmpl.ModTime().After(fiGoEmbed.ModTime()) } +// isExtensionMarkup return true if the file extension ext match with one of +// supported markup format. func isExtensionMarkup(ext string) bool { - return ext == extAsciidoc + if ext == extAsciidoc { + return true + } + return ext == extMarkdown } // listFileMarkups find any markup files inside the content directory, // recursively. func listFileMarkups(dir string, excRE []*regexp.Regexp) ( - fileMarkups map[string]*fileMarkup, err error, + fileMarkups map[string]*FileMarkup, err error, ) { var ( logp = `listFileMarkups` d *os.File fi os.FileInfo - fmarkup *fileMarkup + fmarkup *FileMarkup name string filePath string k string ext string fis []os.FileInfo - fmarkups map[string]*fileMarkup + fmarkups map[string]*FileMarkup ) d, err = os.Open(dir) @@ -278,7 +285,7 @@ func listFileMarkups(dir string, excRE []*regexp.Regexp) ( return nil, fmt.Errorf(`%s: %w`, logp, err) } - fileMarkups = make(map[string]*fileMarkup) + fileMarkups = make(map[string]*FileMarkup) for _, fi = range fis { name = fi.Name() @@ -317,7 +324,7 @@ func listFileMarkups(dir string, excRE []*regexp.Regexp) ( if fi.Size() == 0 { continue } - fmarkup, err = newFileMarkup(filePath, fi) + fmarkup, err = NewFileMarkup(filePath, fi) if err != nil { return nil, fmt.Errorf(`%s: %s: %w`, logp, filePath, err) } diff --git a/ciigo_test.go b/ciigo_test.go index 64becf0..1efb6f4 100644 --- a/ciigo_test.go +++ b/ciigo_test.go @@ -21,6 +21,7 @@ func TestListFileMarkups(t *testing.T) { excRegex: `(ex)/.*`, exp: []string{ `testdata/in/clu/de/file.adoc`, + `testdata/in/clu/de/markdown.md`, }, }, { excRegex: `(in|ex)/.*`, @@ -30,7 +31,9 @@ func TestListFileMarkups(t *testing.T) { excRegex: `file$`, exp: []string{ `testdata/ex/clu/de/file.adoc`, + `testdata/ex/clu/de/markdown.md`, `testdata/in/clu/de/file.adoc`, + `testdata/in/clu/de/markdown.md`, }, }} @@ -39,7 +42,7 @@ func TestListFileMarkups(t *testing.T) { c testCase excre *regexp.Regexp - list map[string]*fileMarkup + list map[string]*FileMarkup got []string k string err error diff --git a/cmd/ciigo-example/static.go b/cmd/ciigo-example/static.go index aa2c1a6..f2561db 100644 --- a/cmd/ciigo-example/static.go +++ b/cmd/ciigo-example/static.go @@ -200,6 +200,7 @@ func init() { }, Excludes: []string{ `.*\.adoc$`, + `.*\.md$`, `^\..*`, }, Embed: memfs.EmbedOptions{ diff --git a/converter.go b/converter.go index 9fcbe97..58f9974 100644 --- a/converter.go +++ b/converter.go @@ -6,11 +6,15 @@ package ciigo import ( "fmt" "html/template" + "io/ioutil" "log" "os" "path/filepath" "git.sr.ht/~shulhan/asciidoctor-go" + "github.com/yuin/goldmark" + meta "github.com/yuin/goldmark-meta" + "github.com/yuin/goldmark/parser" ) // Converter a single, reusable AsciiDoc converter. @@ -62,11 +66,11 @@ func NewConverter(htmlTemplate string) (converter *Converter, err error) { } // convertFileMarkups convert markup files into HTML. -func (converter *Converter) convertFileMarkups(fileMarkups map[string]*fileMarkup, isForce bool) { +func (converter *Converter) convertFileMarkups(fileMarkups map[string]*FileMarkup, isForce bool) { var ( logp = `convertFileMarkups` - fmarkup *fileMarkup + fmarkup *FileMarkup err error ) @@ -77,7 +81,7 @@ func (converter *Converter) convertFileMarkups(fileMarkups map[string]*fileMarku } } - err = converter.ToHtmlFile(fmarkup.path, fmarkup.pathHtml) + err = converter.ToHtmlFile(fmarkup) if err != nil { log.Printf(`%s: %s`, logp, err) } else { @@ -105,32 +109,27 @@ func (converter *Converter) SetHtmlTemplateFile(pathHtmlTemplate string) (err er } // ToHtmlFile convert the AsciiDoc file to HTML. -func (converter *Converter) ToHtmlFile(pathAdoc, pathHtml string) (err error) { +func (converter *Converter) ToHtmlFile(fmarkup *FileMarkup) (err error) { var ( - logp = `ToHtmlFile` - fhtml = newFileHtml() + logp = `ToHtmlFile` - htmlBody string - doc *asciidoctor.Document - f *os.File + fhtml *fileHtml + f *os.File ) - doc, err = asciidoctor.Open(pathAdoc) - if err != nil { - return fmt.Errorf(`%s: %w`, logp, err) + switch fmarkup.kind { + case markupKindAdoc: + fhtml, err = converter.adocToHtml(fmarkup) + case markupKindMarkdown: + fhtml, err = converter.markdownToHtml(fmarkup) } - - err = doc.ToHTMLBody(&fhtml.rawBody) if err != nil { return fmt.Errorf(`%s: %w`, logp, err) } - fhtml.unpackAdocMetadata(doc) - - htmlBody = fhtml.rawBody.String() - fhtml.Body = template.HTML(htmlBody) + fhtml.Body = template.HTML(fhtml.rawBody.String()) - f, err = os.Create(pathHtml) + f, err = os.Create(fmarkup.pathHtml) if err != nil { return fmt.Errorf(`%s: %w`, logp, err) } @@ -147,3 +146,57 @@ func (converter *Converter) ToHtmlFile(pathAdoc, pathHtml string) (err error) { return nil } + +func (converter *Converter) adocToHtml(fmarkup *FileMarkup) (fhtml *fileHtml, err error) { + var ( + logp = `adocToHtml` + doc *asciidoctor.Document + ) + + doc, err = asciidoctor.Open(fmarkup.path) + if err != nil { + return nil, fmt.Errorf(`%s: %w`, logp, err) + } + + fhtml = newFileHtml() + + err = doc.ToHTMLBody(&fhtml.rawBody) + if err != nil { + return nil, fmt.Errorf(`%s: %w`, logp, err) + } + + fhtml.unpackAdocMetadata(doc) + + return fhtml, nil +} + +func (converter *Converter) markdownToHtml(fmarkup *FileMarkup) (fhtml *fileHtml, err error) { + var ( + logp = `markdownToHtml` + mdg = goldmark.New( + goldmark.WithExtensions( + meta.Meta, + ), + ) + + in []byte + parserCtx parser.Context + ) + + in, err = ioutil.ReadFile(fmarkup.path) + if err != nil { + return nil, fmt.Errorf(`%s: %w`, logp, err) + } + + fhtml = newFileHtml() + parserCtx = parser.NewContext() + + err = mdg.Convert(in, &fhtml.rawBody, parser.WithContext(parserCtx)) + if err != nil { + return nil, fmt.Errorf(`%s: %w`, logp, err) + } + + fhtml.unpackMarkdownMetadata(meta.Get(parserCtx)) + + return fhtml, nil +} diff --git a/filemarkup.go b/file_markup.go index e0eb994..131b8f7 100644 --- a/filemarkup.go +++ b/file_markup.go @@ -10,17 +10,28 @@ import ( "strings" ) -type fileMarkup struct { +// List of markup kind. +const ( + markupKindAdoc = 1 + markupKindMarkdown = 2 +) + +// FileMarkup contains the markup path and its kind. +type FileMarkup struct { info os.FileInfo // info contains FileInfo of markup file. - basePath string // basePath contains full path to file without markup extension. - path string // path contains full path to markup file. + basePath string // Full path to file without markup extension. + path string // Full path to markup file. pathHtml string // path to HTML file. + + kind int } -func newFileMarkup(filePath string, fi os.FileInfo) (fmarkup *fileMarkup, err error) { +// NewFileMarkup create new FileMarkup instance form file in "filePath". +// The "fi" option is optional, if its nil it will Stat-ed manually. +func NewFileMarkup(filePath string, fi os.FileInfo) (fmarkup *FileMarkup, err error) { var ( - logp = `newFileMarkup` + logp = `NewFileMarkup` ext string ) @@ -37,10 +48,11 @@ func newFileMarkup(filePath string, fi os.FileInfo) (fmarkup *fileMarkup, err er ext = strings.ToLower(filepath.Ext(filePath)) - fmarkup = &fileMarkup{ + fmarkup = &FileMarkup{ path: filePath, info: fi, basePath: strings.TrimSuffix(filePath, ext), + kind: markupKind(ext), } fmarkup.pathHtml = fmarkup.basePath + `.html` @@ -49,7 +61,7 @@ func newFileMarkup(filePath string, fi os.FileInfo) (fmarkup *fileMarkup, err er } // isNewerThanHtml return true if the markup file is newer than HTML file. -func (fm *fileMarkup) isNewerThanHtml() bool { +func (fm *FileMarkup) isNewerThanHtml() bool { var ( fi os.FileInfo ) @@ -59,3 +71,13 @@ func (fm *fileMarkup) isNewerThanHtml() bool { } return fm.info.ModTime().After(fi.ModTime()) } + +func markupKind(ext string) int { + switch ext { + case extAsciidoc: + return markupKindAdoc + case extMarkdown: + return markupKindMarkdown + } + return 0 +} diff --git a/filehtml.go b/filehtml.go index 4c3ea1c..235329e 100644 --- a/filehtml.go +++ b/filehtml.go @@ -4,6 +4,7 @@ package ciigo import ( + "fmt" "html/template" "strings" @@ -12,6 +13,7 @@ import ( const ( metadataStylesheet = `stylesheet` + metadataTitle = `title` ) // fileHtml represent an HTML metadata for header and its body. @@ -58,3 +60,37 @@ func (fhtml *fileHtml) unpackAdocMetadata(doc *asciidoctor.Document) { fhtml.EmbeddedCSS = embeddedCSS() } } + +func (fhtml *fileHtml) unpackMarkdownMetadata(metadata map[string]any) { + var ( + key string + val any + vstr string + ok bool + ) + + fhtml.Styles = fhtml.Styles[:0] + + for key, val = range metadata { + vstr, ok = val.(string) + if !ok { + vstr = fmt.Sprintf(`%s`, val) + } + + key = strings.ToLower(key) + switch key { + case metadataStylesheet: + fhtml.Styles = append(fhtml.Styles, vstr) + case metadataTitle: + fhtml.Title = vstr + default: + // Metadata `author_names`, `description`, + // `generator`, and `keywords` goes here. + fhtml.Metadata[key] = vstr + } + } + + if len(fhtml.Styles) == 0 { + fhtml.EmbeddedCSS = embeddedCSS() + } +} @@ -8,11 +8,14 @@ go 1.18 require ( git.sr.ht/~shulhan/asciidoctor-go v0.4.1 github.com/shuLhan/share v0.44.0 + github.com/yuin/goldmark v1.5.4 + github.com/yuin/goldmark-meta v1.1.0 ) require ( golang.org/x/net v0.7.0 // indirect golang.org/x/sys v0.5.0 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect ) //replace git.sr.ht/~shulhan/asciidoctor-go => ../asciidoctor-go @@ -2,7 +2,15 @@ git.sr.ht/~shulhan/asciidoctor-go v0.4.1 h1:Zev0L5HyMjH43sPaoJal8E/Hmbel/akoGOxN git.sr.ht/~shulhan/asciidoctor-go v0.4.1/go.mod h1:vRHDUl3o3UzDkvVR9dEFYQ0JDqOh0TKpOZWvOh/CGZU= github.com/shuLhan/share v0.44.0 h1:Afom8pQrzNYtUZM53y+eqlZw5lkFm7bgl3QjZ3ARsgg= github.com/shuLhan/share v0.44.0/go.mod h1:BnjohSsgDFMeYQ0/ws7kzb1oABZdVbEwDNGbUhOLee4= +github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= +github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc= +github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/testdata/ex/clu/de/markdown.md b/testdata/ex/clu/de/markdown.md new file mode 100644 index 0000000..21a3a9c --- /dev/null +++ b/testdata/ex/clu/de/markdown.md @@ -0,0 +1 @@ +# markdown diff --git a/testdata/in/clu/de/markdown.md b/testdata/in/clu/de/markdown.md new file mode 100644 index 0000000..21a3a9c --- /dev/null +++ b/testdata/in/clu/de/markdown.md @@ -0,0 +1 @@ +# markdown diff --git a/testdata/watcher_test.txt b/testdata/watcher_test.txt new file mode 100644 index 0000000..a1937b9 --- /dev/null +++ b/testdata/watcher_test.txt @@ -0,0 +1,29 @@ +>>> create.md +--- +Title: a title +--- +# watch create + +<<< create.md.html +<!DOCTYPE> +<html> +<head><title>a title</title></head> +<body><h1>watch create</h1> +</body> +</html> + +>>> update.md +--- +Title: A new title +stylesheet: /path/to/style.css +keywords: ciigo,markdown +--- +# watch updated + +<<< update.md.html +<!DOCTYPE> +<html> +<head><title>A new title</title></head> +<body><h1>watch updated</h1> +</body> +</html> @@ -25,7 +25,7 @@ type watcher struct { // fileMarkups contains all markup files found inside "dir". // Its used to convert all markup files when the template file // changes. - fileMarkups map[string]*fileMarkup + fileMarkups map[string]*FileMarkup dir string } @@ -58,6 +58,7 @@ func newWatcher(converter *Converter, convertOpts *ConvertOptions) (w *watcher, Root: convertOpts.Root, Includes: []string{ `.*\.adoc$`, + `.*\.md$`, }, Excludes: []string{ `^\..*`, @@ -106,7 +107,7 @@ func (w *watcher) watchFileMarkup() { logp = `watchFileMarkup` ns memfs.NodeState - fmarkup *fileMarkup + fmarkup *FileMarkup ext string err error ok bool @@ -130,7 +131,7 @@ func (w *watcher) watchFileMarkup() { case memfs.FileStateCreated: fmt.Printf("%s: %s created\n", logp, ns.Node.SysPath) - fmarkup, err = newFileMarkup(ns.Node.SysPath, nil) + fmarkup, err = NewFileMarkup(ns.Node.SysPath, nil) if err != nil { log.Printf("%s: %s\n", logp, err) continue @@ -148,7 +149,7 @@ func (w *watcher) watchFileMarkup() { if fmarkup == nil { log.Printf("%s: %s not found\n", logp, ns.Node.SysPath) - fmarkup, err = newFileMarkup(ns.Node.SysPath, nil) + fmarkup, err = NewFileMarkup(ns.Node.SysPath, nil) if err != nil { log.Printf("%s: %s\n", logp, err) continue @@ -158,7 +159,7 @@ func (w *watcher) watchFileMarkup() { } } - err = w.converter.ToHtmlFile(fmarkup.path, fmarkup.pathHtml) + err = w.converter.ToHtmlFile(fmarkup) if err != nil { log.Printf(`%s: %s`, logp, err) } diff --git a/watcher_test.go b/watcher_test.go index 6b16223..d527309 100644 --- a/watcher_test.go +++ b/watcher_test.go @@ -64,14 +64,33 @@ func TestWatcher(t *testing.T) { t.Fatal(err) } + var tdata *test.Data + + tdata, err = test.LoadData(`testdata/watcher_test.txt`) + if err != nil { + t.Fatal(err) + } + t.Run(`createAdocFile`, testCreate) t.Run(`updateAdocFile`, testUpdate) t.Run(`deleteAdocFile`, testDelete) + + var pathFileMarkdown = filepath.Join(testWatcher.dir, `test.md`) + + t.Run(`testMarkdownCreate`, func(tt *testing.T) { + testMarkdownCreate(tt, tdata, pathFileMarkdown) + }) + t.Run(`testMarkdownUpdate`, func(tt *testing.T) { + testMarkdownUpdate(tt, tdata, pathFileMarkdown) + }) + t.Run(`testMarkdownDelete`, func(tt *testing.T) { + testMarkdownDelete(tt, pathFileMarkdown) + }) } func testCreate(t *testing.T) { var ( - got *fileMarkup + got *FileMarkup err error expBody string gotBody []byte @@ -114,7 +133,7 @@ func testUpdate(t *testing.T) { err error expBody string gotBody []byte - got *fileMarkup + got *FileMarkup ) _, err = testAdocFile.WriteString(`= Hello`) @@ -151,7 +170,7 @@ func testUpdate(t *testing.T) { func testDelete(t *testing.T) { var ( err error - got *fileMarkup + got *FileMarkup gotIsExist bool ) @@ -172,6 +191,91 @@ func testDelete(t *testing.T) { test.Assert(t, `adoc file deleted`, false, gotIsExist) } +func testMarkdownCreate(t *testing.T, tdata *test.Data, pathFileMarkdown string) { + var ( + body = tdata.Input[`create.md`] + + got *FileMarkup + err error + expBody string + gotBody []byte + ) + + // Let the OS sync the file system before we create new file, + // otherwise the modtime for fs.Root does not changes. + time.Sleep(1 * time.Second) + + err = os.WriteFile(pathFileMarkdown, body, 0600) + if err != nil { + t.Fatal(err) + } + + got = waitChanges() + + test.Assert(t, `New md file created`, pathFileMarkdown, got.path) + + gotBody, err = os.ReadFile(got.pathHtml) + if err != nil { + t.Fatal(err) + } + gotBody = removeFooter(gotBody) + + expBody = string(tdata.Output[`create.md.html`]) + test.Assert(t, `HTML body`, expBody, string(gotBody)) +} + +func testMarkdownUpdate(t *testing.T, tdata *test.Data, pathFileMarkdown string) { + var ( + body = tdata.Input[`update.md`] + + got *FileMarkup + err error + expBody string + gotBody []byte + ) + + // Let the OS sync the file system before we create new file, + // otherwise the modtime for fs.Root does not changes. + time.Sleep(1 * time.Second) + + err = os.WriteFile(pathFileMarkdown, body, 0600) + if err != nil { + t.Fatal(err) + } + + got = waitChanges() + + test.Assert(t, `changes path`, pathFileMarkdown, got.path) + + gotBody, err = os.ReadFile(got.pathHtml) + if err != nil { + t.Fatal(err) + } + gotBody = removeFooter(gotBody) + + expBody = string(tdata.Output[`update.md.html`]) + test.Assert(t, `HTML body`, expBody, string(gotBody)) +} + +func testMarkdownDelete(t *testing.T, pathFileMarkdown string) { + var ( + err error + got *FileMarkup + gotIsExist bool + ) + + err = os.Remove(pathFileMarkdown) + if err != nil { + t.Fatal(err) + } + + got = waitChanges() + test.Assert(t, `md file updated`, pathFileMarkdown, got.path) + + _, gotIsExist = testWatcher.fileMarkups[pathFileMarkdown] + test.Assert(t, `md file deleted`, false, gotIsExist) +} + // removeFooter remove the footer from generated HTML since its contains date // and time that changes during test. func removeFooter(in []byte) (out []byte) { @@ -186,13 +290,13 @@ func removeFooter(in []byte) (out []byte) { return out } -func waitChanges() (fmarkup *fileMarkup) { +func waitChanges() (fmarkup *FileMarkup) { var ( ok bool ) for { - fmarkup, ok = testWatcher.changes.Pop().(*fileMarkup) + fmarkup, ok = testWatcher.changes.Pop().(*FileMarkup) if ok { break } |
