For file_system source, user can define what kinds of data they want as metadata

README.md

@@ -104,11 +104,12 @@ takes in a display type and renders the resources in that format.
 
 **This tool has three major commands available**
 
-| Command                  | Description                                                                         |
-|--------------------------|-------------------------------------------------------------------------------------|
-| discover                 | It will go through each sources and discover resources and save it to your database |
-| relation                 | This command is responsible to build relationship between resources                 |
-| display                  | It will generate visualized resource graph that can be opened in your browser       |
+| Command  | Description                                                                         |
+|----------|-------------------------------------------------------------------------------------|
+| discover | It will go through each sources and discover resources and save it to your database |
+| relation | This command is responsible to build relationship between resources                 |
+| display  | It will generate visualized resource graph that can be opened in your browser       |
+| validate | Validate your configuration file                                                    |
 
 ### 📥 Run Discovery
 
@@ -136,6 +137,15 @@ teredix display --config {your_config.yaml file}
 ```
 It will show `Displaying resource graph at http://localhost:8989`. Open your browser and visit the URL.
 
+### Validate configuration file
+
+To test & validate your configuration YAML file,
+
+```shell
+teredix validate --config {your_config.yaml file}
+```
+It will show `Configuration is valid!` or not. If configuration is not valid, it will display error.
+
 ## :computer: Usage
 
 ```shell
@@ -352,14 +362,17 @@ source:
     type: file_system
     configuration:
       root_directory: "/path/to/directory"
+    fields:
+      - rootDirectory
+      - machineHost
 ```
 #### Available metadata for file_system source
 
-| Meta Key       | Description                                       |
-|----------------|---------------------------------------------------|
-| Machine-Host   | Hostname of the machine                           |
-| Root-Directory | Root directory to scan the recursive file list    |
-| Scanner-Label  | Name of the source configured in config.yaml file |
+| Data Fields   | Description                                       |
+|---------------|---------------------------------------------------|
+| machineHost   | Hostname of the machine                           |
+| rootDirectory | Root directory to scan the recursive file list    |
+| Scanner-Label | Name of the source configured in config.yaml file |
 
 
 ## Supported Storage

pkg/cmd/testdata/valid_config.yaml

@@ -26,10 +26,16 @@ source:
     type: file_system
     configuration:
       root_directory: "/some/path"
+    fields:
+      - machineHost
+      - rootDirectory
   fs_two:
     type: file_system
     configuration:
       root_directory: "/some/other/path"
+    fields:
+      - rootDirectory
+      - machineHost
   #two:
   #  type: kubernetes
   #  configuration:
@@ -41,9 +47,9 @@ relations:
     - name: "file-system-rule1"
       source:
         kind: "FilePath"
-        meta_key: "Root-Directory"
+        meta_key: "rootDirectory"
         meta_value: "/some/path"
       target:
         kind: "FilePath"
-        meta_key: "Root-Directory"
+        meta_key: "rootDirectory"
         meta_value: "/some/path"

pkg/cmd/validate_test.go

@@ -14,7 +14,7 @@ func Test_validate(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name:    "valid config file",
+			name:    "invalid config file",
 			cfgFile: "testdata/invalid_config.yaml",
 			wantErr: true,
 		},

pkg/config/config.go

@@ -39,6 +39,7 @@ type Source struct {
 	Type          string            `yaml:"type"`
 	ConfigFrom    string            `yaml:"config_from,omitempty"`
 	Configuration map[string]string `yaml:"configuration"`
+	Fields        []string          `yaml:"fields"`
 	DependsOn     []string          `yaml:"depends_on,omitempty"`
 }
 

pkg/config/schema.json

@@ -195,9 +195,12 @@
         },
         "configuration": {
           "$ref": "#/definitions/source.file_system.configuration"
+        },
+        "fields": {
+          "$ref": "#/definitions/source.file_system.fields"
         }
       },
-      "required": ["type", "configuration"]
+      "required": ["type", "configuration", "fields"]
     },
     "source.file_system.configuration": {
       "type": "object",
@@ -209,6 +212,15 @@
       },
       "required": ["root_directory"]
     },
+    "source.file_system.fields": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": ["rootDirectory", "machineHost"]
+      },
+      "uniqueItems": true,
+      "additionalItems": false
+    },
     "source.github_repository": {
       "type": "object",
       "properties": {

pkg/config/schema_validator_test.go

@@ -41,10 +41,14 @@ source:
     type: file_system
     configuration:
       root_directory: "/some/path"
+    fields: &file_system_fields
+      - rootDirectory
+      - machineHost
   fs_two:
     type: file_system
     configuration:
       root_directory: "/some/other/path"
+    fields: *file_system_fields
   aws_s3_one:
     type: aws_s3
     configuration: &aws_conf
@@ -67,11 +71,11 @@ relations:
     - name: "file-system-rule1"
       source:
         kind: "FilePath"
-        meta_key: "Root-Directory"
+        meta_key: "rootDirectory"
         meta_value: "/some/path"
       target:
         kind: "FilePath"
-        meta_key: "Root-Directory"
+        meta_key: "rootDirectory"
         meta_value: "/some/path"
 `,
 			expectError: false,

pkg/config/testdata/valid_config.yaml

@@ -25,11 +25,10 @@ source:
   fs_one:
     type: file_system
     configuration:
-      root_directory: "/some/path"
-  fs_two:
-    type: file_system
-    configuration:
-      root_directory: "/some/other/path"
+      root_directory: "/home/shaharia/Projects/hellodev"
+    fields:
+      - machineHost
+      - rootDirectory
   aws_s3_one:
     type: aws_s3
     configuration: &aws_conf
@@ -52,9 +51,9 @@ relations:
     - name: "file-system-rule1"
       source:
         kind: "FilePath"
-        meta_key: "Root-Directory"
+        meta_key: "rootDirectory"
         meta_value: "/some/path"
       target:
         kind: "FilePath"
-        meta_key: "Root-Directory"
+        meta_key: "rootDirectory"
         meta_value: "/some/path"

pkg/source/scanner/fs_scanner.go

@@ -11,11 +11,16 @@ import (
 	"github.com/shahariaazam/teredix/pkg/util"
 )
 
+const (
+	fileSystemFieldRootDirectory = "rootDirectory"
+	fileSystemFieldMachineHost   = "machineHost"
+)
+
 // FsScanner store configuration for file system scanner
 type FsScanner struct {
 	name          string
 	rootDirectory string
-	metaData      map[string]string
+	fields        []string
 }
 
 // File represent file information
@@ -24,8 +29,8 @@ type File struct {
 }
 
 // NewFsScanner construct new file system scanner
-func NewFsScanner(name, rootDirectory string, metaData map[string]string) FsScanner {
-	return FsScanner{name: name, rootDirectory: rootDirectory, metaData: metaData}
+func NewFsScanner(name, rootDirectory string, fields []string) FsScanner {
+	return FsScanner{name: name, rootDirectory: rootDirectory, fields: fields}
 }
 
 // Scan scans the file system
@@ -41,27 +46,33 @@ func (s *FsScanner) Scan(resourceChannel chan resource.Resource) error {
 	}
 
 	rootResource := resource.NewResource("FileDirectory", util.GenerateUUID(), s.rootDirectory, s.rootDirectory, s.name)
-	for k, v := range s.metaData {
-		rootResource.AddMetaData(k, v)
+
+	if util.IsFieldExistsInConfig(fileSystemFieldRootDirectory, s.fields) {
+		rootResource.AddMetaData(fileSystemFieldRootDirectory, s.rootDirectory)
+	}
+
+	if util.IsFieldExistsInConfig(fileSystemFieldMachineHost, s.fields) {
+		rootResource.AddMetaData(fileSystemFieldMachineHost, hostname)
 	}
 
-	rootResource.AddMetaData("Machine-Host", hostname)
-	rootResource.AddMetaData("Root-Directory", s.rootDirectory)
 	rootResource.AddMetaData(pkg.MetaKeyScannerLabel, s.name)
 
 	resourceChannel <- rootResource
 
 	for _, f := range files {
 		nr := resource.NewResource("FilePath", util.GenerateUUID(), f.Path, f.Path, s.name)
 		nr.AddRelation(rootResource)
-		for k, v := range s.metaData {
-			nr.AddMetaData(k, v)
-		}
 
-		nr.AddMetaData("Machine-Host", hostname)
-		nr.AddMetaData("Root-Directory", s.rootDirectory)
 		nr.AddMetaData("Scanner-Label", s.name)
 
+		if util.IsFieldExistsInConfig(fileSystemFieldRootDirectory, s.fields) {
+			nr.AddMetaData(fileSystemFieldRootDirectory, s.rootDirectory)
+		}
+
+		if util.IsFieldExistsInConfig(fileSystemFieldMachineHost, s.fields) {
+			nr.AddMetaData(fileSystemFieldMachineHost, hostname)
+		}
+
 		resourceChannel <- nr
 	}