ADD: json schema support for frontmatter

go.mod

@@ -5,7 +5,6 @@ go 1.18
 require (
 	github.com/JoshVarga/svgparser v0.0.0-20200804023048-5eaba627a7d1
 	github.com/adrg/frontmatter v0.2.0
-	github.com/antchfx/jsonquery v1.1.5
 	github.com/antchfx/xmlquery v1.3.3
 	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/cheggaaa/pb/v3 v3.0.8
@@ -16,11 +15,10 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/rogpeppe/go-internal v1.10.0
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
-	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/zyxar/image2ascii v0.0.0-20180912034614-460a04e371ae
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 	golang.org/x/net v0.14.0
@@ -45,8 +43,6 @@ require (
 	github.com/rivo/uniseg v0.4.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
-	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

go.sum

@@ -7,8 +7,6 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd4=
 github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE=
-github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE=
-github.com/antchfx/jsonquery v1.1.5/go.mod h1:RtMzTHohKaAerkfslTNjr3Y9MdxjKlSgIgaVjVKNiug=
 github.com/antchfx/xmlquery v1.3.3 h1:HYmadPG0uz8CySdL68rB4DCLKXz2PurCjS3mnkVF4CQ=
 github.com/antchfx/xmlquery v1.3.3/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
 github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
@@ -81,8 +79,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
-github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
-github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
@@ -91,17 +89,9 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
-github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/zyxar/image2ascii v0.0.0-20180912034614-460a04e371ae h1:EiqxsQwk1eimsz+ncJrsMMMwnkYTGiVOrLe5lGxL9cs=
 github.com/zyxar/image2ascii v0.0.0-20180912034614-460a04e371ae/go.mod h1:Md4Hcw0pmYWDCo1o/fHeOC2Gdhc6oDRwLim8V+SMvI0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

internal/command/frontmatter.go

@@ -12,20 +12,21 @@ import (
 )
 
 var (
-	fmStrict     bool
-	fmStrictSet  map[string]bool
-	fmSorted     bool
-	fmRequired   []string
-	fmOptional   []string
-	fmForbidden  []string
-	fmDelimiters []string
+	fmSchemaOptions shared.SchemaOptions
+	fmStrict        bool
+	fmStrictSet     map[string]bool
+	fmSorted        bool
+	fmRequired      []string
+	fmOptional      []string
+	fmForbidden     []string
+	fmDelimiters    []string
 )
 var frontmatterCmd = &cobra.Command{
 	Args:    cobra.MinimumNArgs(1),
 	Use:     "frontmatter [options] files...",
 	Short:   "Validate frontmatter",
 	Long:    `Checks that the frontmatter in your files is valid`,
-	PreRunE: frontmatterPrepare,
+	PreRunE: frontmatterInit,
 	RunE:    shared.MakeFileCommand(frontmatterCheck),
 }
 
@@ -39,8 +40,9 @@ func AddFrontmatterCommand(rootCmd *cobra.Command) {
 	frontmatterCmd.Flags().BoolVar(&fmSorted, "sorted", false, "Keys need to be in alphabetical order")
 	frontmatterCmd.Flags().StringSliceVar(&fmDelimiters, "delimiters", []string{}, "Custom delimiters (if other than `---`, `+++` and `;;;`)")
 
+	fmSchemaOptions.AddFlags(frontmatterCmd)
+
 	//LATER: report
-	//LATER: schema
 }
 
 func frontmatterCheck(f *shared.FileContext) {
@@ -53,7 +55,7 @@ func frontmatterCheck(f *shared.FileContext) {
 		return
 	}
 
-	yamlData := make(map[interface{}]interface{})
+	yamlData := make(map[string]any)
 
 	var formats []*frontmatter.Format
 
@@ -68,6 +70,7 @@ func frontmatterCheck(f *shared.FileContext) {
 		}
 	}
 
+	//LATER: maybe flag to require contents?
 	_, parseErr := frontmatter.MustParse(bytes.NewReader(data), &yamlData, formats...)
 
 	f.RecordResult("frontmatterParse", parseErr == nil, map[string]interface{}{
@@ -97,17 +100,9 @@ func frontmatterCheck(f *shared.FileContext) {
 
 	if fmStrict {
 		for key := range yamlData {
-			keyStr, strErr := key.(string)
-			if !strErr {
-				f.RecordResult("frontmatterStrictParse", false, map[string]interface{}{
-					"err": "key is not a string",
-					"key": fmt.Sprintf("%v", key),
-				})
-				continue
-			}
-			_, ok := fmStrictSet[keyStr]
+			_, ok := fmStrictSet[key]
 			f.RecordResult("frontmatterStrict", ok, map[string]interface{}{
-				"key": keyStr,
+				"key": key,
 			})
 		}
 	}
@@ -133,9 +128,11 @@ func frontmatterCheck(f *shared.FileContext) {
 			previousKey = currentKey
 		}
 	}
+
+	fmSchemaOptions.Validate(f, yamlData)
 }
 
-func frontmatterPrepare(cmd *cobra.Command, args []string) error {
+func frontmatterInit(cmd *cobra.Command, args []string) error {
 	if fmStrict {
 		fmStrictSet = make(map[string]bool)
 		for _, key := range fmRequired {
@@ -150,5 +147,11 @@ func frontmatterPrepare(cmd *cobra.Command, args []string) error {
 		fmt.Fprintf(os.Stderr, "ERROR: delimiter count must be <=2 (passed %d)", len(fmDelimiters))
 		os.Exit(7)
 	}
+
+	schemaPrepErr := fmSchemaOptions.Prepare()
+	if schemaPrepErr != nil {
+		return schemaPrepErr
+	}
+
 	return nil
 }

internal/command/json.go

@@ -1,21 +1,16 @@
 package command
 
 import (
-	"bytes"
-	"fmt"
-	"net/url"
-	"os"
-	"path/filepath"
+	"encoding/json"
 
 	"github.com/FileFormatInfo/fflint/internal/shared"
-	"github.com/antchfx/jsonquery"
+	//"github.com/antchfx/jsonquery"
 	"github.com/spf13/cobra"
-	"github.com/xeipuuv/gojsonschema"
 )
 
 var (
-	jsonSchemaLocation string
-	jsonSchema         *gojsonschema.Schema
+	jsonSchemaValidator shared.SchemaOptions
+	jsonSchemaLocation  string
 )
 
 // jsonCmd represents the json command
@@ -31,10 +26,8 @@ var jsonCmd = &cobra.Command{
 func AddJsonCommand(rootCmd *cobra.Command) {
 	rootCmd.AddCommand(jsonCmd)
 
-	jsonCmd.Flags().StringVar(&jsonSchemaLocation, "schema", "", "JSON Schema to validate against") //LATER: link to docs about embedded ones
-
+	jsonSchemaValidator.AddFlags(jsonCmd)
 	//LATER: whitespace: canonical/none/any
-	//LATER: schema (https://github.com/xeipuuv/gojsonschema)
 }
 
 func jsonCheck(f *shared.FileContext) {
@@ -47,59 +40,25 @@ func jsonCheck(f *shared.FileContext) {
 		return
 	}
 
-	_, parseErr := jsonquery.Parse(bytes.NewReader(data))
+	var jsonData any
+	parseErr := json.Unmarshal(data, &jsonData)
 
+	f.RecordResult("jsonParse", parseErr == nil, map[string]interface{}{
+		"error": parseErr,
+	})
 	if parseErr != nil {
-		f.RecordResult("jsonParse", false, map[string]interface{}{
-			"error": parseErr,
-		})
 		return
 	}
 
-	if jsonSchema != nil {
-		result, validateErr := jsonSchema.Validate(gojsonschema.NewStringLoader(string(data)))
-		if validateErr != nil {
-			f.RecordResult("jsonSchemaRun", false, map[string]interface{}{
-				"error": validateErr.Error(),
-			})
-		} else {
-			f.RecordResult("jsonSchemaValidate", result.Valid(), map[string]interface{}{
-				"errors": result.Errors(),
-			})
-		}
-	}
-
+	jsonSchemaValidator.Validate(f, jsonData)
 }
 
 func jsonInit(cmd *cobra.Command, args []string) error {
 
-	if jsonSchemaLocation == "" {
-		return nil
-	}
-
-	// work with local file urls
-	jsonUrl, urlParseErr := url.Parse(jsonSchemaLocation)
-	if urlParseErr != nil {
-		return urlParseErr
-	}
-
-	// allow relative local file schemas
-	if jsonUrl.Scheme == "" {
-		jsonUrl.Scheme = "file"
-		jsonPath, pathErr := filepath.Abs(jsonUrl.Path)
-		if pathErr != nil {
-			return pathErr
-		}
-		jsonUrl.Path = jsonPath
-		newLocation := jsonUrl.String()
-		if shared.Debug {
-			fmt.Fprintf(os.Stderr, "DEBUG: canonicalizing schema path from '%s' to '%s'\n", jsonSchemaLocation, newLocation)
-		}
-		jsonSchemaLocation = newLocation
+	prepErr := jsonSchemaValidator.Prepare()
+	if prepErr != nil {
+		return prepErr
 	}
 
-	jsonSchemaLoader := gojsonschema.NewReferenceLoader(jsonSchemaLocation)
-	var schemaErr error
-	jsonSchema, schemaErr = gojsonschema.NewSchema(jsonSchemaLoader)
-	return schemaErr
+	return nil
 }

internal/shared/schema.go

@@ -1,6 +1,9 @@
 package shared
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/santhosh-tekuri/jsonschema/v5"
 	"github.com/spf13/cobra"
 )
@@ -11,20 +14,43 @@ type SchemaOptions struct {
 }
 
 func (so *SchemaOptions) AddFlags(theCmd *cobra.Command) {
-
 	theCmd.Flags().StringVar(&so.src, "schema", "", "JSON schema path (or URL)")
 }
 
 func (so *SchemaOptions) Prepare() error {
+	if Debug {
+		fmt.Fprintf(os.Stderr, "DEBUG: trying to compile schema: %s\n", so.src)
+	}
+	if so.src == "" {
+		return nil
+	}
 	compiled, err := jsonschema.Compile(so.src)
 	if err != nil {
+		if Debug {
+			fmt.Fprintf(os.Stderr, "DEBUG: schema compilation error: %v\n", err)
+		}
 		return err
 	}
 
+	if Debug {
+		fmt.Fprintf(os.Stderr, "DEBUG: successfully compiled schema from %s\n", so.src)
+	}
+
 	so.compiled = compiled
 	return nil
 }
 
-func (so *SchemaOptions) Validate(data interface{}) error {
-	return so.compiled.Validate(data)
+func (so *SchemaOptions) Validate(f *FileContext, data any) error {
+	if so.compiled == nil {
+		return nil
+	}
+	validationErr := so.compiled.Validate(data)
+	if Debug {
+		fmt.Fprintf(os.Stderr, "DEBUG: schema validation error: %v\n", validationErr)
+	}
+	f.RecordResult("jsonSchemaValidatation", validationErr == nil, map[string]interface{}{
+		"error": validationErr,
+	})
+
+	return validationErr
 }

run.sh

@@ -1,14 +1,29 @@
 #!/usr/bin/env bash
 #
-# run locally
+# run tests locally
 #
 
 set -o errexit
 set -o pipefail
 set -o nounset
 
-rm -rf ./fflint
+if [ -f "./fflint" ]; then
+  echo "INFO: removing old build of fflint"
+  rm ./fflint
+fi
+
+echo "INFO: building new fflint"
 go build -o ./fflint cmd/fflint/main.go
-export PATH=$PATH:$(pwd)
-fflint version
-go test -timeout 30s -run "^TestFflint$" github.com/FileFormatInfo/fflint/cmd/fflint
+if [ ! -f "./fflint" ]; then
+  echo "ERROR: failed to build fflint"
+  exit 1
+fi
+
+
+export PATH=$(pwd):$PATH
+echo "INFO: running fflint version $(fflint version)"
+
+echo "INFO: running tests"
+go test -timeout 30s -run "^TestFflint$" github.com/FileFormatInfo/fflint/cmd/fflint
+
+echo "INFO: complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)"