diff --git a/ci/src/Registry/API.purs b/ci/src/Registry/API.purs index 05d1ad60f..3e4f3b32e 100644 --- a/ci/src/Registry/API.purs +++ b/ci/src/Registry/API.purs @@ -28,7 +28,7 @@ import Registry.PackageUpload as Upload import Registry.RegistryM (Env, RegistryM, closeIssue, comment, commitToTrunk, readPackagesMetadata, runRegistryM, throwWithComment, updatePackagesMetadata, uploadPackage) import Registry.Schema (Manifest, Metadata, Operation(..), Repo(..), addVersionToMetadata, mkNewMetadata) import Registry.Scripts.LegacyImport as LegacyImport -import Registry.Scripts.LegacyImport.Bowerfile as Bowerfile +import Registry.Types (bowerToManifestFields) import Sunde as Process import Text.Parsing.StringParser as StringParser @@ -151,7 +151,7 @@ addOrUpdate { ref, fromBower, packageName } metadata = do let absoluteTarballPath = tmpDir <> "/" <> tarballName let archiveUrl = "https://github.com/" <> owner <> "/" <> repo <> "/archive/" <> tarballName log $ "Fetching tarball from GitHub: " <> archiveUrl - wget archiveUrl absoluteTarballPath + wget archiveUrl absoluteTarballPath ------------------------------ we have the git repo log $ "Tarball downloaded in " <> absoluteTarballPath liftEffect (Tar.getToplevelDir absoluteTarballPath) >>= case _ of Nothing -> @@ -179,7 +179,7 @@ addOrUpdate { ref, fromBower, packageName } metadata = do Json.stringifyWithIndent 2 <<< Json.encodeJson <<< NEA.toArray manifestFields = - Bowerfile.toManifestFields bowerfile + bowerToManifestFields bowerfile runManifest = Except.runExceptT <<< Except.mapExceptT (liftAff <<< map (lmap printErrors)) diff --git a/ci/src/Registry/Error.purs b/ci/src/Registry/Error.purs new file mode 100644 index 000000000..4ded4be74 --- /dev/null +++ b/ci/src/Registry/Error.purs @@ -0,0 +1,10 @@ +module Registry.Error where + +import Prelude + +import Data.Array.NonEmpty (NonEmptyArray) +import Data.Array.NonEmpty as NEA +import Data.Either (Either(..)) + +mkError :: forall e a. e -> Either (NonEmptyArray e) a +mkError = Left <<< NEA.singleton \ No newline at end of file diff --git a/ci/src/Registry/License.purs b/ci/src/Registry/License.purs new file mode 100644 index 000000000..178a015dd --- /dev/null +++ b/ci/src/Registry/License.purs @@ -0,0 +1,64 @@ +module Registry.License (LicenseError(..), produceLicense) where + +import Prelude + +import Data.Argonaut as Json +import Data.Argonaut.Decode.Generic as Json.Decode.Generic +import Data.Argonaut.Encode.Generic as Json.Encode.Generic +import Data.Argonaut.Types.Generic as Json.Generic +import Data.Array.NonEmpty (NonEmptyArray) +import Data.Array.NonEmpty as NEA +import Data.Either (Either(..)) +import Data.Filterable (filter) +import Data.Generic.Rep (class Generic) +import Data.Maybe (Maybe(..)) +import Data.String.NonEmpty (NonEmptyString) +import Data.String.NonEmpty as NES +import Foreign.SPDX (License) +import Foreign.SPDX as SPDX +import Registry.Prelude (genericShow, partitionEithers) + +data LicenseError + = MissingLicense + | BadLicense (Array String) + +derive instance eqLicenseError :: Eq LicenseError +derive instance ordLicenseError :: Ord LicenseError +derive instance genericLicenseError :: Generic LicenseError _ +instance showLicenseError :: Show LicenseError where + show e = genericShow e + +instance Json.EncodeJson LicenseError where + encodeJson = Json.Encode.Generic.genericEncodeJsonWith (Json.Generic.defaultEncoding { unwrapSingleArguments = true }) + +instance Json.DecodeJson LicenseError where + decodeJson = Json.Decode.Generic.genericDecodeJsonWith (Json.Generic.defaultEncoding { unwrapSingleArguments = true }) + +produceLicense :: Maybe (NonEmptyArray NonEmptyString) -> Either LicenseError License +produceLicense license = do + let + rewrite = case _ of + "Apache 2" -> "Apache-2.0" + "Apache-2" -> "Apache-2.0" + "Apache 2.0" -> "Apache-2.0" + "BSD" -> "BSD-3-Clause" + "BSD3" -> "BSD-3-Clause" + "BSD-3" -> "BSD-3-Clause" + "3-Clause BSD" -> "BSD-3-Clause" + other -> other + + case license of + Nothing -> Left MissingLicense + Just licenses -> do + let + parsed = + map (SPDX.parse <<< rewrite) + $ filter (_ /= "LICENSE") + $ map NES.toString + $ NEA.toArray licenses + { fail, success } = partitionEithers parsed + + case fail, success of + [], [] -> Left MissingLicense + [], _ -> Right $ SPDX.joinWith SPDX.Or success + _, _ -> Left $ BadLicense fail \ No newline at end of file diff --git a/ci/src/Registry/Manifest.purs b/ci/src/Registry/Manifest.purs new file mode 100644 index 000000000..09deea90e --- /dev/null +++ b/ci/src/Registry/Manifest.purs @@ -0,0 +1,176 @@ +module Registry.Manifest where + +import Registry.Prelude + +import Affjax as Http +import Affjax.ResponseFormat as ResponseFormat +import Affjax.StatusCode (StatusCode(..)) +import Control.Monad.Except as Except +import Control.Parallel (parallel, sequential) +import Data.Argonaut as Json +import Data.Array as Array +import Data.Array.NonEmpty as NEA +import Data.Compactable (compact) +import Data.Interpolate (i) +import Data.Maybe (maybe) +import Data.String as String +import Data.String.NonEmpty as NES +import Foreign.Dhall as Dhall +import Foreign.GitHub as GitHub +import Foreign.Jsonic as Jsonic +import Foreign.Licensee as Licensee +import Foreign.Tmp as Tmp +import Node.FS.Aff as FS +import Registry.Process as Process +import Registry.Types (FileResource(..), ImportError(..), ManifestFields, RawPackageName(..), RawVersion(..), RemoteResource(..), RequestError(..), SpagoJson, bowerToManifestFields, fileResourcePath, spagoToManifestFields) + +-- | Attempt to construct the basic fields necessary for a manifest file by reading +-- | the package version's bower.json, spago.dhall, package.json, and LICENSE +-- | files, if present. +constructManifestFields + :: RawPackageName + -> RawVersion + -> GitHub.Address + -> ExceptT ImportError Aff ManifestFields +constructManifestFields package version address = do + let cacheKey = i "manifest-fields__" (un RawPackageName package) "__" (un RawVersion version) + Process.withCache Process.jsonSerializer cacheKey Nothing do + -- We can construct a manifest from a package's bowerfile, package.json file, + -- spago.dhall file, and/or LICENSE files. A package doesn't need to have all + -- of these files; several of these files duplicate information. We try to + -- fetch all files but won't throw an exception (yet) if they're missing. + log $ "Constructing manifest fields for " <> un RawPackageName package <> " " <> un RawVersion version + let mkRequest file = parallel $ Except.runExceptT $ fileRequest file Process.stringSerializer + files <- liftAff $ sequential ado + licenseFile <- mkRequest LicenseFile + bowerJson <- mkRequest BowerJson + packageJson <- mkRequest PackageJson + in { licenseFile, bowerJson, packageJson } + + -- TODO: Improve this heuristic by checking the Bower _and_ Spago files. + -- + -- We can pull dependencies from the bower.json or spago.dhall files. If both + -- files are present, but their dependencies differ, then we should use the + -- file with newer dependencies; presumably, it's the more up-to-date file. + -- + -- Since Bower users typically use ranges, but package sets use precise + -- versions, we could check to see whether one uses later major versions + -- than the other does; checking minor or patch versions will be inaccurate. + -- + -- If the files differ but it isn't clear which file is newer, then we should + -- prefer the Bower file since it's the legacy format used for package p + -- publishing. + -- + -- For now, that's exactly what we do: use the Bower file if it is present, + -- and otherwise fall back to the Spago file. + bowerManifest <- Except.runExceptT do + result <- Except.except files.bowerJson + case Jsonic.parseJson result >>= Json.decodeJson of + Left err -> do + let printed = Json.printJsonDecodeError err + log $ "Could not decode returned bower.json. " <> printed + log result + throwError $ ResourceError { resource: FileResource BowerJson, error: DecodeError printed } + Right bowerfile -> + pure $ bowerToManifestFields bowerfile + + spagoJson <- liftAff $ Except.runExceptT requestSpagoJson + let spagoManifest = map spagoToManifestFields spagoJson + + { dependencies, devDependencies } <- case bowerManifest, spagoManifest of + Left _, Left _ -> do + -- TODO: We may want to report a `NonEmptyArray ImportError` so as to + -- report on multiple errors, such as the multiple missing files in this + -- situation. + throwError NoDependencyFiles + Left _, Right { dependencies, devDependencies } -> + pure { dependencies, devDependencies } + Right { dependencies, devDependencies }, _ -> + pure { dependencies, devDependencies } + + -- We can detect the license for the project using a combination of `licensee` + -- and reading the license directly out of the Spago and Bower files (the + -- CLI tool will not read from either file). + licenseeOutput <- detectLicense files ------------------ pull this out + + let + spagoLicenses = maybe [] NEA.toArray $ _.license =<< hush spagoManifest + bowerLicenses = maybe [] NEA.toArray $ _.license =<< hush bowerManifest + licenseeLicenses = compact $ map NES.fromString licenseeOutput + license = NEA.fromArray $ Array.nub $ Array.concat [ licenseeLicenses, spagoLicenses, bowerLicenses ] + + when (license == Nothing) do + log $ "No license available for " <> un RawPackageName package <> " " <> un RawVersion version + + pure { license, dependencies, devDependencies } + where + detectLicense { licenseFile, packageJson } = do + licenseeResult <- liftAff $ Licensee.detectFiles $ compact $ map hush + -- Detection only works on these files, and won't work on Spago files, + -- Bower files, or the JSON produced by the dhall-to-json result of + -- converting the Spago file. + [ packageJson <#> { name: "package.json", contents: _ } + , licenseFile <#> { name: "LICENSE", contents: _ } + ] + + detectedLicenses <- case licenseeResult of + Left err -> do + log $ "Licensee decoding error, ignoring: " <> err + pure [] + Right licenses -> + pure licenses + + pure detectedLicenses + + -- Attempt to construct a Spago JSON file by fetching the spago.dhall and + -- packages.dhall files and converting them to JSON with dhall-to-json. + requestSpagoJson :: ExceptT ImportError Aff SpagoJson + requestSpagoJson = do + files <- sequential ado + spagoDhall <- parallel $ fileRequest SpagoDhall Process.stringSerializer + packagesDhall <- parallel $ fileRequest PackagesDhall Process.stringSerializer + in { spagoDhall, packagesDhall } + + tmp <- liftEffect Tmp.mkTmpDir + liftAff $ FS.writeTextFile UTF8 (tmp <> "/packages.dhall") files.packagesDhall + + spagoJson <- do + let + mkError = ResourceError <<< { resource: FileResource SpagoDhall, error: _ } <<< DecodeError + runDhallJson = Dhall.dhallToJson { dhall: files.spagoDhall, cwd: Just tmp } + + Except.mapExceptT (map (lmap mkError)) + $ Except.ExceptT + $ map (_ >>= (Json.decodeJson >>> lmap Json.printJsonDecodeError)) runDhallJson + + pure spagoJson + + -- Request a file from the remote repository associated with the package + -- version. Files will be cached using the provided serializer and + -- will be read from the cache up to the cache expiry time given in `Hours`. + fileRequest :: FileResource -> Process.Serialize String String -> ExceptT ImportError Aff String + fileRequest resource serialize = do + let + name = un RawPackageName package + tag = un RawVersion version + filePath = fileResourcePath resource + url = i "https://raw.githubusercontent.com/" address.owner "/" address.repo "/" tag "/" filePath + fileCacheName = String.replace (String.Pattern ".") (String.Replacement "-") filePath + cacheKey = i fileCacheName "__" name "__" tag + mkError = ResourceError <<< { resource: FileResource resource, error: _ } + + Process.withCache serialize cacheKey Nothing do + liftAff (Http.get ResponseFormat.string url) >>= case _ of + Left error -> do + let printed = Http.printError error + log $ i "Unable to retrieve " filePath " because the request failed: " printed + throwError $ mkError BadRequest + Right { status: StatusCode status, body } + | status == 404 -> do + log $ i "Unable to retrieve " filePath " because none exists (404 error)." + throwError $ mkError $ BadStatus status + | status /= 200 -> do + log $ i "Unable to retrieve " filePath " because of a bad status code: " body + throwError $ mkError $ BadStatus status + | otherwise -> + pure body diff --git a/ci/src/Registry/Scripts/LegacyImport/Process.purs b/ci/src/Registry/Process.purs similarity index 95% rename from ci/src/Registry/Scripts/LegacyImport/Process.purs rename to ci/src/Registry/Process.purs index 1f05bcfd2..3353e2670 100644 --- a/ci/src/Registry/Scripts/LegacyImport/Process.purs +++ b/ci/src/Registry/Process.purs @@ -1,4 +1,4 @@ -module Registry.Scripts.LegacyImport.Process where +module Registry.Process where import Registry.Prelude @@ -22,8 +22,8 @@ import Foreign.SemVer (SemVer) import Node.FS.Aff as FS import Node.FS.Stats (Stats(..)) import Registry.PackageName (PackageName) -import Registry.Scripts.LegacyImport.Error (ImportError(..), ImportErrorKey, PackageFailures(..), RawPackageName, RawVersion, RequestError(..)) -import Registry.Scripts.LegacyImport.Error as LegacyImport.Error +import Registry.Types (ImportError(..), ImportErrorKey, PackageFailures(..), RawPackageName, RawVersion, RequestError(..)) +import Registry.Types as Registry.Types type ProcessedPackages k a = { failures :: PackageFailures @@ -47,7 +47,7 @@ forPackage input f = do Except.runExceptT (f name value) >>= case _ of Left err -> do let - errorType = LegacyImport.Error.printImportErrorKey err + errorType = Registry.Types.printImportErrorKey err failure = Map.singleton name (Left err) var # modifyAVar \state -> state { failures = insertFailure errorType failure state.failures } Right (Tuple newKey result) -> do @@ -83,7 +83,7 @@ forPackageVersion input f = do Except.runExceptT (f k1 k2 value) >>= case _ of Left err -> do let - errorType = LegacyImport.Error.printImportErrorKey err + errorType = Registry.Types.printImportErrorKey err failure = Map.singleton name $ Right $ Map.singleton tag err var # modifyAVar \state -> state { failures = insertFailure errorType failure state.failures } Right result -> do @@ -124,7 +124,7 @@ forPackageVersionKeys input f = do Except.runExceptT (f k1 tag) >>= case _ of Left err -> do let - errorType = LegacyImport.Error.printImportErrorKey err + errorType = Registry.Types.printImportErrorKey err failure = Map.singleton name $ Right $ Map.singleton tag err var # modifyAVar \state -> state { failures = insertFailure errorType failure state.failures } Right (Tuple k3 k4) -> do diff --git a/ci/src/Registry/Scripts/LegacyImport.purs b/ci/src/Registry/Scripts/LegacyImport.purs index 83c847671..379fc5da6 100644 --- a/ci/src/Registry/Scripts/LegacyImport.purs +++ b/ci/src/Registry/Scripts/LegacyImport.purs @@ -2,48 +2,35 @@ module Registry.Scripts.LegacyImport where import Registry.Prelude -import Affjax as Http -import Affjax.ResponseFormat as ResponseFormat -import Affjax.StatusCode (StatusCode(..)) import Control.Monad.Except as Except -import Control.Parallel (parallel, sequential) import Data.Argonaut as Json import Data.Array (catMaybes) import Data.Array as Array import Data.Array.NonEmpty as NEA -import Data.Interpolate (i) import Data.Lens (_Left, preview) import Data.Map as Map -import Data.Maybe (maybe) import Data.Monoid (guard) import Data.Set as Set import Data.String as String -import Data.String.NonEmpty as NES import Data.Time.Duration (Hours(..)) import Dotenv as Dotenv import Effect.Aff as Aff import Effect.Class.Console (logShow) -import Foreign.Dhall as Dhall import Foreign.GitHub as GitHub -import Foreign.Jsonic as Jsonic -import Foreign.Licensee as Licensee import Foreign.Object as Object -import Foreign.SPDX as SPDX import Foreign.SemVer (SemVer) import Foreign.SemVer as SemVer -import Foreign.Tmp as Tmp import Node.FS.Aff as FS +import Registry.Error (mkError) import Registry.Index (RegistryIndex, insertManifest) +import Registry.License (produceLicense) +import Registry.Manifest (constructManifestFields) import Registry.PackageName (PackageName) import Registry.PackageName as PackageName import Registry.Schema (Repo(..), Manifest) -import Registry.Scripts.LegacyImport.Bowerfile as Bowerfile -import Registry.Scripts.LegacyImport.Error (APIResource(..), FileResource(..), ImportError(..), ManifestError(..), PackageFailures(..), RawPackageName(..), RawVersion(..), RemoteResource(..), RequestError(..), fileResourcePath) -import Registry.Scripts.LegacyImport.ManifestFields (ManifestFields) -import Registry.Scripts.LegacyImport.Process as Process -import Registry.Scripts.LegacyImport.SpagoJson (SpagoJson) -import Registry.Scripts.LegacyImport.SpagoJson as SpagoJson +import Registry.Process as Process import Registry.Scripts.LegacyImport.Stats as Stats +import Registry.Types (APIResource(..), ImportError(..), ManifestError(..), ManifestFields, PackageFailures(..), RawPackageName(..), RawVersion(..), RemoteResource(..), RequestError(..)) import Safe.Coerce (coerce) import Text.Parsing.StringParser as StringParser @@ -195,37 +182,7 @@ toManifest -> ExceptT (NonEmptyArray ManifestError) Aff Manifest toManifest package repository version manifest = do let - mkError :: forall a. ManifestError -> Either (NonEmptyArray ManifestError) a - mkError = Left <<< NEA.singleton - - eitherLicense = do - let - rewrite = case _ of - "Apache 2" -> "Apache-2.0" - "Apache-2" -> "Apache-2.0" - "Apache 2.0" -> "Apache-2.0" - "BSD" -> "BSD-3-Clause" - "BSD3" -> "BSD-3-Clause" - "BSD-3" -> "BSD-3-Clause" - "3-Clause BSD" -> "BSD-3-Clause" - other -> other - - case manifest.license of - Nothing -> mkError MissingLicense - Just licenses -> do - let - parsed = - map (SPDX.parse <<< rewrite) - $ Array.filter (_ /= "LICENSE") - $ map NES.toString - $ NEA.toArray licenses - { fail, success } = partitionEithers parsed - - case fail, success of - [], [] -> mkError MissingLicense - [], _ -> Right $ SPDX.joinWith SPDX.Or success - _, _ -> mkError $ BadLicense fail - + eitherLicense = lmap (NEA.singleton <<< LicenseError) (produceLicense manifest.license) eitherTargets = do let -- We trim out packages that don't begin with `purescript-`, as these @@ -329,154 +286,3 @@ readRegistryFile source = do Right packages -> do let toPackagesArray = Object.toArrayWithKey \k -> Tuple (RawPackageName $ stripPureScriptPrefix k) pure $ Map.fromFoldable $ toPackagesArray packages - --- | Attempt to construct the basic fields necessary for a manifest file by reading --- | the package version's bower.json, spago.dhall, package.json, and LICENSE --- | files, if present. -constructManifestFields - :: RawPackageName - -> RawVersion - -> GitHub.Address - -> ExceptT ImportError Aff ManifestFields -constructManifestFields package version address = do - let cacheKey = i "manifest-fields__" (un RawPackageName package) "__" (un RawVersion version) - Process.withCache Process.jsonSerializer cacheKey Nothing do - -- We can construct a manifest from a package's bowerfile, package.json file, - -- spago.dhall file, and/or LICENSE files. A package doesn't need to have all - -- of these files; several of these files duplicate information. We try to - -- fetch all files but won't throw an exception (yet) if they're missing. - log $ "Constructing manifest fields for " <> un RawPackageName package <> " " <> un RawVersion version - let mkRequest file = parallel $ Except.runExceptT $ fileRequest file Process.stringSerializer - files <- liftAff $ sequential ado - licenseFile <- mkRequest LicenseFile - bowerJson <- mkRequest BowerJson - packageJson <- mkRequest PackageJson - in { licenseFile, bowerJson, packageJson } - - -- TODO: Improve this heuristic by checking the Bower _and_ Spago files. - -- - -- We can pull dependencies from the bower.json or spago.dhall files. If both - -- files are present, but their dependencies differ, then we should use the - -- file with newer dependencies; presumably, it's the more up-to-date file. - -- - -- Since Bower users typically use ranges, but package sets use precise - -- versions, we could check to see whether one uses later major versions - -- than the other does; checking minor or patch versions will be inaccurate. - -- - -- If the files differ but it isn't clear which file is newer, then we should - -- prefer the Bower file since it's the legacy format used for package p - -- publishing. - -- - -- For now, that's exactly what we do: use the Bower file if it is present, - -- and otherwise fall back to the Spago file. - bowerManifest <- Except.runExceptT do - result <- Except.except files.bowerJson - case Jsonic.parseJson result >>= Json.decodeJson of - Left err -> do - let printed = Json.printJsonDecodeError err - log $ "Could not decode returned bower.json. " <> printed - log result - throwError $ ResourceError { resource: FileResource BowerJson, error: DecodeError printed } - Right bowerfile -> - pure $ Bowerfile.toManifestFields bowerfile - - spagoJson <- liftAff $ Except.runExceptT requestSpagoJson - let spagoManifest = map SpagoJson.toManifestFields spagoJson - - { dependencies, devDependencies } <- case bowerManifest, spagoManifest of - Left _, Left _ -> do - -- TODO: We may want to report a `NonEmptyArray ImportError` so as to - -- report on multiple errors, such as the multiple missing files in this - -- situation. - throwError NoDependencyFiles - Left _, Right { dependencies, devDependencies } -> - pure { dependencies, devDependencies } - Right { dependencies, devDependencies }, _ -> - pure { dependencies, devDependencies } - - -- We can detect the license for the project using a combination of `licensee` - -- and reading the license directly out of the Spago and Bower files (the - -- CLI tool will not read from either file). - licenseeOutput <- detectLicense files - - let - spagoLicenses = maybe [] NEA.toArray $ _.license =<< hush spagoManifest - bowerLicenses = maybe [] NEA.toArray $ _.license =<< hush bowerManifest - licenseeLicenses = Array.catMaybes $ map NES.fromString licenseeOutput - license = NEA.fromArray $ Array.nub $ Array.concat [ licenseeLicenses, spagoLicenses, bowerLicenses ] - - when (license == Nothing) do - log $ "No license available for " <> un RawPackageName package <> " " <> un RawVersion version - - pure { license, dependencies, devDependencies } - where - detectLicense { licenseFile, packageJson } = do - licenseeResult <- liftAff $ Licensee.detectFiles $ Array.catMaybes $ map hush - -- Detection only works on these files, and won't work on Spago files, - -- Bower files, or the JSON produced by the dhall-to-json result of - -- converting the Spago file. - [ packageJson <#> { name: "package.json", contents: _ } - , licenseFile <#> { name: "LICENSE", contents: _ } - ] - - detectedLicenses <- case licenseeResult of - Left err -> do - log $ "Licensee decoding error, ignoring: " <> err - pure [] - Right licenses -> - pure licenses - - pure detectedLicenses - - -- Attempt to construct a Spago JSON file by fetching the spago.dhall and - -- packages.dhall files and converting them to JSON with dhall-to-json. - requestSpagoJson :: ExceptT ImportError Aff SpagoJson - requestSpagoJson = do - files <- sequential ado - spagoDhall <- parallel $ fileRequest SpagoDhall Process.stringSerializer - packagesDhall <- parallel $ fileRequest PackagesDhall Process.stringSerializer - in { spagoDhall, packagesDhall } - - tmp <- liftEffect Tmp.mkTmpDir - liftAff $ FS.writeTextFile UTF8 (tmp <> "/packages.dhall") files.packagesDhall - - spagoJson <- do - let - mkError = ResourceError <<< { resource: FileResource SpagoDhall, error: _ } <<< DecodeError - runDhallJson = Dhall.dhallToJson { dhall: files.spagoDhall, cwd: Just tmp } - - Except.mapExceptT (map (lmap mkError)) - $ Except.ExceptT - $ map (_ >>= (Json.decodeJson >>> lmap Json.printJsonDecodeError)) runDhallJson - - pure spagoJson - - -- Request a file from the remote repository associated with the package - -- version. Files will be cached using the provided serializer and - -- will be read from the cache up to the cache expiry time given in `Hours`. - fileRequest :: FileResource -> Process.Serialize String String -> ExceptT ImportError Aff String - fileRequest resource serialize = do - let - name = un RawPackageName package - tag = un RawVersion version - filePath = fileResourcePath resource - url = i "https://raw.githubusercontent.com/" address.owner "/" address.repo "/" tag "/" filePath - fileCacheName = String.replace (String.Pattern ".") (String.Replacement "-") filePath - cacheKey = i fileCacheName "__" name "__" tag - mkError = ResourceError <<< { resource: FileResource resource, error: _ } - - Process.withCache serialize cacheKey Nothing do - liftAff (Http.get ResponseFormat.string url) >>= case _ of - Left error -> do - let printed = Http.printError error - log $ i "Unable to retrieve " filePath " because the request failed: " printed - throwError $ mkError BadRequest - Right { status: StatusCode status, body } - | status == 404 -> do - log $ i "Unable to retrieve " filePath " because none exists (404 error)." - throwError $ mkError $ BadStatus status - | status /= 200 -> do - log $ i "Unable to retrieve " filePath " because of a bad status code: " body - throwError $ mkError $ BadStatus status - | otherwise -> - pure body diff --git a/ci/src/Registry/Scripts/LegacyImport/Bowerfile.purs b/ci/src/Registry/Scripts/LegacyImport/Bowerfile.purs deleted file mode 100644 index d19496cb1..000000000 --- a/ci/src/Registry/Scripts/LegacyImport/Bowerfile.purs +++ /dev/null @@ -1,46 +0,0 @@ -module Registry.Scripts.LegacyImport.Bowerfile - ( Bowerfile(..) - , toManifestFields - ) where - -import Registry.Prelude - -import Control.Alt ((<|>)) -import Data.Argonaut (Json, (.:?)) -import Data.Argonaut as Json -import Data.Array as Array -import Data.Array.NonEmpty as NEA -import Data.String.NonEmpty (NonEmptyString) -import Data.String.NonEmpty as NES -import Registry.Scripts.LegacyImport.ManifestFields (ManifestFields) - -toManifestFields :: Bowerfile -> ManifestFields -toManifestFields (Bowerfile fields) = fields - -newtype Bowerfile = Bowerfile ManifestFields - -derive newtype instance Eq Bowerfile -derive newtype instance Show Bowerfile -derive newtype instance Json.EncodeJson Bowerfile - -instance Json.DecodeJson Bowerfile where - decodeJson json = do - obj <- Json.decodeJson json - license <- decodeStringOrStringArray obj "license" - dependencies <- fromMaybe mempty <$> obj .:? "dependencies" - devDependencies <- fromMaybe mempty <$> obj .:? "devDependencies" - pure $ Bowerfile { license, dependencies, devDependencies } - -decodeStringOrStringArray - :: Object Json - -> String - -> Either Json.JsonDecodeError (Maybe (NonEmptyArray NonEmptyString)) -decodeStringOrStringArray obj fieldName = do - let typeError = const $ Json.AtKey fieldName $ Json.TypeMismatch "String or Array" - lmap typeError do - value <- obj .:? fieldName - case value of - Nothing -> pure Nothing - Just v -> do - decoded <- (Json.decodeJson v <#> Array.singleton) <|> Json.decodeJson v - pure $ NEA.fromArray $ Array.catMaybes $ map NES.fromString decoded diff --git a/ci/src/Registry/Scripts/LegacyImport/ManifestFields.purs b/ci/src/Registry/Scripts/LegacyImport/ManifestFields.purs deleted file mode 100644 index 2f20cbd91..000000000 --- a/ci/src/Registry/Scripts/LegacyImport/ManifestFields.purs +++ /dev/null @@ -1,11 +0,0 @@ -module Registry.Scripts.LegacyImport.ManifestFields where - -import Registry.Prelude - -import Data.String.NonEmpty (NonEmptyString) - -type ManifestFields = - { license :: Maybe (NonEmptyArray NonEmptyString) - , dependencies :: Object String - , devDependencies :: Object String - } diff --git a/ci/src/Registry/Scripts/LegacyImport/SpagoJson.purs b/ci/src/Registry/Scripts/LegacyImport/SpagoJson.purs deleted file mode 100644 index a3b31d224..000000000 --- a/ci/src/Registry/Scripts/LegacyImport/SpagoJson.purs +++ /dev/null @@ -1,66 +0,0 @@ -module Registry.Scripts.LegacyImport.SpagoJson - ( SpagoJson - , toManifestFields - ) where - -import Registry.Prelude - -import Data.Argonaut ((.:?)) -import Data.Argonaut as Json -import Data.Array as Array -import Data.Array.NonEmpty as NEA -import Data.Map as Map -import Data.String.NonEmpty (NonEmptyString) -import Data.String.NonEmpty as NES -import Foreign.Object as Object -import Registry.Scripts.LegacyImport.Error (RawPackageName(..), RawVersion(..)) -import Registry.Scripts.LegacyImport.ManifestFields (ManifestFields) - -toManifestFields :: SpagoJson -> ManifestFields -toManifestFields spago@(SpagoJson { license }) = - { license: map NEA.singleton license - , dependencies: packageDependencies spago - , devDependencies: Object.empty - } - -packageDependencies :: SpagoJson -> Object String -packageDependencies (SpagoJson { dependencies, packages }) = do - let - foldFn m name = fromMaybe m do - version <- Map.lookup name packages - pure $ Object.insert (un RawPackageName name) (un RawVersion version) m - - Array.foldl foldFn Object.empty dependencies - --- | The output of calling `dhall-to-json` on a `spago.dhall` file -newtype SpagoJson = SpagoJson - { license :: Maybe NonEmptyString - , dependencies :: Array RawPackageName - , packages :: Map RawPackageName RawVersion - } - -derive newtype instance Eq SpagoJson - -instance Json.EncodeJson SpagoJson where - encodeJson (SpagoJson spago) = do - let - packagesMap = map { version: _ } spago.packages - packagesObject = objectFromMap (un RawPackageName) packagesMap - - Json.encodeJson - { license: spago.license - , dependencies: spago.dependencies - , packages: packagesObject - } - -instance Json.DecodeJson SpagoJson where - decodeJson json = do - obj <- Json.decodeJson json - license' <- obj .:? "license" - dependencies <- fromMaybe mempty <$> obj .:? "dependencies" - packageObj :: Object { version :: RawVersion } <- fromMaybe Object.empty <$> obj .:? "packages" - let - packagesMap = objectToMap (Just <<< RawPackageName) packageObj - packages = map _.version packagesMap - license = NES.fromString =<< license' - pure $ SpagoJson { license, dependencies, packages } diff --git a/ci/src/Registry/Scripts/LegacyImport/Stats.purs b/ci/src/Registry/Scripts/LegacyImport/Stats.purs index 26c4602b3..a9c20924a 100644 --- a/ci/src/Registry/Scripts/LegacyImport/Stats.purs +++ b/ci/src/Registry/Scripts/LegacyImport/Stats.purs @@ -24,8 +24,8 @@ import Foreign.GitHub as GitHub import Foreign.SemVer (SemVer) import Registry.PackageName (PackageName) import Registry.Schema (Manifest) -import Registry.Scripts.LegacyImport.Error (ImportError(..), ImportErrorKey(..), ManifestError, ManifestErrorKey(..), PackageFailures(..), RawPackageName, RawVersion, manifestErrorKey, printManifestErrorKey) -import Registry.Scripts.LegacyImport.Process (ProcessedPackageVersions) +import Registry.Types (ImportError(..), ImportErrorKey(..), ManifestError, ManifestErrorKey(..), PackageFailures(..), RawPackageName, RawVersion, manifestErrorKey, printManifestErrorKey) +import Registry.Process (ProcessedPackageVersions) import Safe.Coerce (coerce) newtype ErrorCounts = ErrorCounts diff --git a/ci/src/Registry/Scripts/LegacyImport/Error.purs b/ci/src/Registry/Types.purs similarity index 66% rename from ci/src/Registry/Scripts/LegacyImport/Error.purs rename to ci/src/Registry/Types.purs index 0f8c11827..f7bc4fd76 100644 --- a/ci/src/Registry/Scripts/LegacyImport/Error.purs +++ b/ci/src/Registry/Types.purs @@ -1,16 +1,31 @@ -module Registry.Scripts.LegacyImport.Error where +module Registry.Types where import Registry.Prelude +import Control.Alt ((<|>)) +import Data.Argonaut (Json, (.:?)) import Data.Argonaut as Json import Data.Argonaut.Decode.Generic as Json.Decode.Generic import Data.Argonaut.Encode.Generic as Json.Encode.Generic import Data.Argonaut.Types.Generic as Json.Generic +import Data.Array as Array +import Data.Array.NonEmpty as NEA import Data.Generic.Rep (class Generic) import Data.Interpolate (i) +import Data.Map as Map +import Data.String.NonEmpty (NonEmptyString) +import Data.String.NonEmpty as NES +import Foreign.Object as Object +import Registry.License as License import Registry.PackageName (PackageName) import Safe.Coerce (coerce) +type ManifestFields = + { license :: Maybe (NonEmptyArray NonEmptyString) + , dependencies :: Object String + , devDependencies :: Object String + } + -- | A map of error types to package names to package versions, where failed -- | versions contain rich information about why they failed. newtype PackageFailures = PackageFailures (Map ImportErrorKey (Map RawPackageName (Either ImportError (Map RawVersion ImportError)))) @@ -112,8 +127,7 @@ instance Json.DecodeJson RequestError where -- | An error representing why a manifest could not be produced for this package data ManifestError = MissingName - | MissingLicense - | BadLicense (Array String) + | LicenseError License.LicenseError | BadVersion String | InvalidDependencyNames (NonEmptyArray String) | BadDependencyVersions (NonEmptyArray { dependency :: PackageName, failedBounds :: String }) @@ -138,8 +152,8 @@ instance Show ManifestErrorKey where printManifestErrorKey :: ManifestError -> ManifestErrorKey printManifestErrorKey = ManifestErrorKey <<< case _ of MissingName -> "missingName" - MissingLicense -> "missingLicense" - BadLicense _ -> "badLicense" + LicenseError License.MissingLicense -> "missingLicense" + LicenseError (License.BadLicense _) -> "badLicense" BadVersion _ -> "badVersion" InvalidDependencyNames _ -> "invalidDependencyNames" BadDependencyVersions _ -> "badDependencyVersions" @@ -196,3 +210,87 @@ fileResourcePath = case _ of PackagesDhall -> "packages.dhall" PackageJson -> "package.json" LicenseFile -> "LICENSE" + +----------- SpagoJson + +spagoToManifestFields :: SpagoJson -> ManifestFields +spagoToManifestFields spago@(SpagoJson { license }) = + { license: map NEA.singleton license + , dependencies: packageDependencies spago + , devDependencies: Object.empty + } + +packageDependencies :: SpagoJson -> Object String +packageDependencies (SpagoJson { dependencies, packages }) = do + let + foldFn m name = fromMaybe m do + version <- Map.lookup name packages + pure $ Object.insert (un RawPackageName name) (un RawVersion version) m + + Array.foldl foldFn Object.empty dependencies + +-- | The output of calling `dhall-to-json` on a `spago.dhall` file +newtype SpagoJson = SpagoJson + { license :: Maybe NonEmptyString + , dependencies :: Array RawPackageName + , packages :: Map RawPackageName RawVersion + } + +derive newtype instance Eq SpagoJson + +instance Json.EncodeJson SpagoJson where + encodeJson (SpagoJson spago) = do + let + packagesMap = map { version: _ } spago.packages + packagesObject = objectFromMap (un RawPackageName) packagesMap + + Json.encodeJson + { license: spago.license + , dependencies: spago.dependencies + , packages: packagesObject + } + +instance Json.DecodeJson SpagoJson where + decodeJson json = do + obj <- Json.decodeJson json + license' <- obj .:? "license" + dependencies <- fromMaybe mempty <$> obj .:? "dependencies" + packageObj :: Object { version :: RawVersion } <- fromMaybe Object.empty <$> obj .:? "packages" + let + packagesMap = objectToMap (Just <<< RawPackageName) packageObj + packages = map _.version packagesMap + license = NES.fromString =<< license' + pure $ SpagoJson { license, dependencies, packages } + +-------------- Bower + +bowerToManifestFields :: Bowerfile -> ManifestFields +bowerToManifestFields (Bowerfile fields) = fields + +newtype Bowerfile = Bowerfile ManifestFields + +derive newtype instance Eq Bowerfile +derive newtype instance Show Bowerfile +derive newtype instance Json.EncodeJson Bowerfile + +instance Json.DecodeJson Bowerfile where + decodeJson json = do + obj <- Json.decodeJson json + license <- decodeStringOrStringArray obj "license" + dependencies <- fromMaybe mempty <$> obj .:? "dependencies" + devDependencies <- fromMaybe mempty <$> obj .:? "devDependencies" + pure $ Bowerfile { license, dependencies, devDependencies } + +decodeStringOrStringArray + :: Object Json + -> String + -> Either Json.JsonDecodeError (Maybe (NonEmptyArray NonEmptyString)) +decodeStringOrStringArray obj fieldName = do + let typeError = const $ Json.AtKey fieldName $ Json.TypeMismatch "String or Array" + lmap typeError do + value <- obj .:? fieldName + case value of + Nothing -> pure Nothing + Just v -> do + decoded <- (Json.decodeJson v <#> Array.singleton) <|> Json.decodeJson v + pure $ NEA.fromArray $ Array.catMaybes $ map NES.fromString decoded diff --git a/ci/test/Main.purs b/ci/test/Main.purs index 9eab27993..5d2073dcc 100644 --- a/ci/test/Main.purs +++ b/ci/test/Main.purs @@ -15,7 +15,7 @@ import Foreign.SemVer as SemVer import Registry.API as API import Registry.PackageName as PackageName import Registry.Schema (Operation(..), Repo(..)) -import Registry.Scripts.LegacyImport.Bowerfile (Bowerfile(..)) +import Registry.Types (Bowerfile(..)) import Test.Foreign.Jsonic (jsonic) import Test.Foreign.Licensee (licensee) import Test.Registry.Index as Registry.Index diff --git a/ci/test/Registry/Scripts/LegacyImport/Stats.purs b/ci/test/Registry/Scripts/LegacyImport/Stats.purs index afd5e8ceb..816fbb492 100644 --- a/ci/test/Registry/Scripts/LegacyImport/Stats.purs +++ b/ci/test/Registry/Scripts/LegacyImport/Stats.purs @@ -12,8 +12,9 @@ import Foreign.SemVer (SemVer, parseSemVer) import Registry.PackageName (PackageName) import Registry.PackageName as PackageName import Registry.Schema (Repo(..)) -import Registry.Scripts.LegacyImport.Error (ImportError(..), ManifestError(..), PackageFailures(..), RawPackageName(..), RawVersion(..), manifestErrorKey, printImportErrorKey, printManifestErrorKey) -import Registry.Scripts.LegacyImport.Process (ProcessedPackageVersions) +import Registry.License as RLicense +import Registry.Types (ImportError(..), ManifestError(..), PackageFailures(..), RawPackageName(..), RawVersion(..), manifestErrorKey, printImportErrorKey, printManifestErrorKey) +import Registry.Process (ProcessedPackageVersions) import Registry.Scripts.LegacyImport.Stats (ErrorCounts(..)) import Registry.Scripts.LegacyImport.Stats as Stats import Test.Spec as Spec @@ -73,8 +74,8 @@ exampleFailures = PackageFailures $ Map.fromFoldable , RawVersion "2.0.0" /\ ManifestError threeManifestErrors ] - twoManifestErrors = MissingLicense :| [ MissingName ] - threeManifestErrors = MissingLicense :| [ BadVersion "x.y.z", InvalidDependencyNames ("doesn't" :| [ "exist" ]) ] + twoManifestErrors = (LicenseError RLicense.MissingLicense) :| [ MissingName ] + threeManifestErrors = (LicenseError RLicense.MissingLicense) :| [ BadVersion "x.y.z", InvalidDependencyNames ("doesn't" :| [ "exist" ]) ] errByPackage = Left errsByVersion = Right @@ -157,7 +158,7 @@ errorStats = do Spec.it "sums the number of each type of import, regardless of which packages or versions it occurred in" do exampleStats.countManifestErrorsByErrorType `Assert.shouldEqual` Map.fromFoldable - [ printManifestErrorKey MissingLicense /\ errCounts 2 1 2 + [ printManifestErrorKey (LicenseError RLicense.MissingLicense) /\ errCounts 2 1 2 , printManifestErrorKey MissingName /\ errCounts 1 1 1 , printManifestErrorKey (BadVersion "") /\ errCounts 1 1 1 , printManifestErrorKey (InvalidDependencyNames (NonEmptyArray.singleton "")) /\ errCounts 1 1 1