Merge pull request #33 from GrantBirki/one-error-per-file-fix

Bug Fix: Report all errors, not just one

__tests__/fixtures/real_world/challenges/challenge.yml

@@ -0,0 +1,88 @@
+# from https://github.com/codosseum-org/challenges/blob/first-challenge/challenges/rotating-numbers/challenge.yml
+# this file intentionally contains two schema errors for unit testing purposes
+# this is a yamlAsJson test
+
+$schema: https://raw.githubusercontent.com/codosseum-org/challenges/main/challenge-schema.json
+author:
+  name: JohnnyJayJay
+  contact:
+    - type: matrix
+      contact: "@johnny:yatrix.org"
+
+license: CC-BY-SA-4.0
+
+language: en
+title: "Rotating Numbers"
+difficulty: "medium"
+tags: ["numbers", "simple-input", "math"]
+text: |
+  Rotate a number given in a specific base by a given amount of digits **to the right**.
+  Rotating to the right means: *lower order* (last) digits become *higher order* (first) digits.
+
+  There are three lines of input:
+  1. The amount of digits to rotate (non-negative)
+  2. The base of the input number (a value between 2 and 16)
+  3. The input number in that base
+
+  The **output** should be the rotated number **converted to decimal format**.
+
+  Be careful:
+  - The output must not contain leading zeros.
+  - The rotation number can be greater than the number of digits of the number.
+  - Any base between 2 and 16 is possible, not just "common" ones.
+
+
+input_format: "TODO"
+
+examples:
+  - in: ["3", "10", "12345"]
+    out: ["34512"]
+  - in: ["1", "2", "1101"]
+    out: ["14"]
+  - in: ["2", "10", "13408"]
+    out: ["8134"]
+  - in: ["3", "10", "13408"]
+    out: ["40813"]
+
+public_tests:
+  - name: Zero Rotation
+    in: ["0", "10", "12345"]
+    out: ["12345"]
+  - name: Simple decimal case
+    in: ["4", "10", "7062545"]
+    out: ["2545706"]
+  - name: Hex
+    in: ["2", "16", "ABCD"]
+    out: ["52651"]
+  - name: Simple binary
+    in: ["1", "2", "101"]
+    out: ["6"]
+  - name: Complex binary
+    in: ["12", "2", "111101101100111"]
+    out: ["23359"]
+  - name: Obscure base
+    in: ["3", "13", "A32B9"]
+    out: ["82943"]
+  - name: Zero
+    in: ["4", "2", "0"]
+    out: ["0"]
+  - name: Leading zero
+    in: ["1", "10", "24351490"]
+    out: ["2435149"]
+  - name: Leading zeros
+    in: ["5", "16", "2A00028"]
+    out: ["164515"]
+  - name: Skipping zeros
+    in: ["5", "8", "2340076"]
+    out: ["1052563"]
+  - name: Single overflow rotation
+    in: ["4", "10", "123"]
+    out: ["312"]
+  - name: Multi overflow rotation
+    in: ["2478395", "10", "123456"]
+    out: ["234561"]
+
+
+solution:
+  language: python
+  file: solution.py

__tests__/fixtures/schemas/challenge.json

@@ -0,0 +1,212 @@
+{
+    "type": "object",
+    "description": "This is the challenge format schema used for our challenge repository.\nChallenges you submit to the codosseum challenges repository must adhere to this schema.",
+    "required": [
+        "license",
+        "language",
+        "title",
+        "difficulty",
+        "text",
+        "inputFormat",
+        "publicTests",
+        "solution"
+    ],
+    "properties": {
+        "author": {
+            "type": "object",
+            "description": "Information about the author of the challenge.",
+            "properties": {
+                "name": {
+                    "type": "string",
+                    "description": "Name of the author",
+                    "examples": [
+                        "Jane Doe",
+                        "ExampleUsername04"
+                    ]
+                },
+                "contact": {
+                    "type": "array",
+                    "description": "(Optional) contact information of the author",
+                    "items": {
+                        "type": "object",
+                        "required": [
+                            "type",
+                            "contact"
+                        ],
+                        "examples": [
+                            {
+                            "type": "email",
+                            "contact": "[email protected]"
+                        },
+                            {
+                            "type": "matrix",
+                            "contact": "@jane:example.com"
+                        },
+                            {
+                            "type": "website",
+                            "contact": "https://user.example.com/blog"
+                        }
+                        ],
+                        "properties": {
+                            "type": {
+                                "type": "string",
+                                "description": "Type of contact information, e.g. \"Email\" or \"Matrix\". This is not restricted to any particular set of values and implementations shouldn't expect it to be."
+                            },
+                            "contact": {
+                                "type": "string",
+                                "description": "The actual contact info"
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "license": {
+            "type": "string",
+            "description": "SPDX license expression to describe the license of the challenge. This MUST be compatible with CC-BY-4.0, CC-BY-SA-4.0 or CC0-1.0.",
+            "examples": [
+                "CC-BY-SA-4.0",
+                "CC-BY-4.0 OR MIT",
+                "CC0-1.0"
+            ]
+        },
+        "language": {
+            "type": "string",
+            "description": "Language of the challenge text and title.\nFor this repository, it is always \"en\", for now.",
+            "const": "en"
+        },
+        "title": {
+            "type": "string",
+            "description": "Title of the challenge",
+            "examples": [
+                "Squaring Circles",
+                "The Universal Answer",
+                "Making PI(e)"
+            ]
+        },
+        "difficulty": {
+            "type": "string",
+            "description": "Difficulty level of the challenge",
+            "enum": ["easy", "medium", "hard"]
+        },
+        "tags": {
+            "type": "array",
+            "description": "Keywords that can be used to put this challenge into certain groups and make it more searchable.",
+            "items": {
+                "type": "string",
+                "examples": [
+                    "string-manipulation",
+                    "math",
+                    "lore"
+                ]
+            }
+        },
+        "text": {
+            "type": "string",
+            "description": "Challenge text. This is the field that has to contain the explanation for the task.",
+            "examples": [
+                "Alice just completed an online coding challenge that just required her to square a number. Fortunately, she could come up with a much harder challenge...",
+                "Bob is on an adventure and has to find his way home...",
+                "Given an input of strings, compute the letter with the highest frequency that..."
+            ]
+        },
+        "inputFormat": {
+            "type": "string",
+            "description": "A formal description of what inputs for this challenge look like.\nThis is required to generate secret tests and solution templates.\nTherefore, if this isn't specified, these things cannot be accomplished.\n\nRight now, the formal language for this field has not been specified yet,\nso it cannot be\n"
+        },
+        "examples": {
+            "type": "array",
+            "description": "Example input-output pairs that illustrate the challenge.\nThey should be fairly short and simple to understand, but can be used to\nhint at certain behavior that may not be immediately obvious.\nA handful of examples should be plenty, typically 2-3 are a good number.\nThey may be used together with public tests for the \"reverse\" game mode.\n",
+            "items": {
+                "$ref": "#/$defs/test"
+            }
+        },
+        "publicTests": {
+            "type": "array",
+            "description": "Tests whose inputs and outputs will be made visible to the user.\nThese should cover most cases of the problem and test the user's code for different flaws. There should be more public tests than examples.\nA public test should have a name, and that name should indicate roughly what it is testing.\n\nTests specified in the examples do not have to be specified again here.\n",
+            "minItems": 5,
+            "items": {
+                "$ref": "#/$defs/test"
+            }
+        },
+        "solution": {
+            "type": "object",
+            "description": "Information about the solution (provided alongside the challenge file) that will be used for generative tests.",
+            "required": [
+                "language",
+                "file"
+            ],
+            "properties": {
+                "language": {
+                    "type": "string",
+                    "description": "Programming language descriptor of the language the solution is written in."
+                },
+                "file": {
+                    "type": "string",
+                    "description": "File location (relative to challenge file) containing the solution. Must be a regular file without any external dependencies."
+                }
+            }
+        }
+    },
+    "$defs": {
+        "test": {
+            "type": "object",
+            "description": "A test describes a function (solution for a challenge) by specifying an output\nthat is expected given an input.\nBoth in- and output are arrays representing lines of text.\n",
+            "required": [
+                "in",
+                "out"
+            ],
+            "examples": [
+                {
+                "name": "Simple case",
+                "in": [
+                    "1",
+                    "2",
+                    "3"
+                ],
+                "out": [
+                    "1",
+                    "2",
+                    "3"
+                ]
+            },
+                {
+                "name": "Edge Case A",
+                "in": [
+                    "banana ananab sillygoose"
+                ],
+                "out": [
+                    "Error!"
+                ]
+            },
+                {
+                "name": "Shhh!",
+                "in": [
+                    "0"
+                ],
+                "out": []
+            }
+            ],
+            "properties": {
+                "name": {
+                    "type": "string",
+                    "title": "(Optional) name of the test"
+                },
+                "in": {
+                    "type": "array",
+                    "title": "lines of input",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "out": {
+                    "type": "array",
+                    "title": "lines of output",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        }
+    }
+}

__tests__/functions/json-validator.test.js

@@ -186,6 +186,10 @@ test('fails to validate one json file with an incorrect schema and succeeds on t
           {
             path: '/foo',
             message: 'must be string'
+          },
+          {
+            path: '/bar',
+            message: 'must be string'
           }
         ]
       }
@@ -254,6 +258,47 @@ test('processes multiple files when yaml_as_json is true and also a mixture of o
   )
 })
 
+test('processes a real world example when yaml_as_json is true and the single file contains multiple schema errors', async () => {
+  process.env.INPUT_YAML_AS_JSON = 'true'
+  process.env.INPUT_JSON_SCHEMA = '__tests__/fixtures/schemas/challenge.json'
+  process.env.INPUT_BASE_DIR = '__tests__/fixtures/real_world/challenges'
+
+  expect(await jsonValidator(excludeMock)).toStrictEqual({
+    failed: 1,
+    passed: 0,
+    skipped: 0,
+    success: false,
+    violations: [
+      {
+        file: '__tests__/fixtures/real_world/challenges/challenge.yml',
+        errors: [
+          {
+            path: null,
+            message: `must have required property 'inputFormat'`
+          },
+          {
+            path: null,
+            message: `must have required property 'publicTests'`
+          }
+        ]
+      }
+    ]
+  })
+
+  expect(debugMock).toHaveBeenCalledWith(
+    'using ajv-formats with json-validator'
+  )
+  expect(debugMock).toHaveBeenCalledWith(
+    'json - using baseDir: __tests__/fixtures/real_world/challenges'
+  )
+  expect(debugMock).toHaveBeenCalledWith(
+    'json - using glob: **/*{.json,yaml,yml}'
+  )
+  expect(debugMock).toHaveBeenCalledWith(
+    `attempting to process yaml file: '__tests__/fixtures/real_world/challenges/challenge.yml' as json`
+  )
+})
+
 test('successfully validates json files with a schema when files is defined', async () => {
   const files = [
     '__tests__/fixtures/json/valid/json1.json',
@@ -288,6 +333,10 @@ test('fails to validate a yaml file with an incorrect schema when yaml_as_json i
           {
             path: null,
             message: "must have required property 'foo'"
+          },
+          {
+            path: null,
+            message: 'must NOT have additional properties'
           }
         ]
       }

src/functions/json-validator.js

@@ -10,7 +10,7 @@ import {parse} from 'yaml'
 // :returns: the compiled schema
 async function schema(jsonSchema) {
   // setup the ajv instance
-  const ajv = new Ajv() // options can be passed, e.g. {allErrors: true}
+  const ajv = new Ajv({allErrors: true}) // options can be passed, e.g. {allErrors: true}
 
   // use ajv-formats if enabled
   if (core.getBooleanInput('use_ajv_formats')) {