From 8d87b534c74913e5422ec179139769717ae7d81f Mon Sep 17 00:00:00 2001 From: Sergey Tihon Date: Tue, 27 Jan 2015 12:45:00 +0300 Subject: [PATCH] paket.lock beautification for HTTP specs --- src/Paket.Core/DependenciesFile.fs | 21 +++--- src/Paket.Core/ModuleResolver.fs | 16 ++-- src/Paket.Core/RemoteDownload.fs | 3 +- .../DependenciesFile/ParserSpecs.fs | 52 ++++++------- tests/Paket.Tests/Lockfile/GeneratorSpecs.fs | 74 ++++++++++++------- .../Resolver/ConflictSourcesSpecs.fs | 10 +-- 6 files changed, 97 insertions(+), 79 deletions(-) diff --git a/src/Paket.Core/DependenciesFile.fs b/src/Paket.Core/DependenciesFile.fs index 66092c62d4..1ba7d71cfd 100644 --- a/src/Paket.Core/DependenciesFile.fs +++ b/src/Paket.Core/DependenciesFile.fs @@ -107,23 +107,24 @@ module DependenciesFileParser = let private ``parse http source`` trimmed = let parts = parseDependencyLine trimmed let getParts (projectSpec:string) fileSpec = - let ``project spec`` = - match projectSpec.EndsWith("/") with - | false -> projectSpec - | true -> projectSpec.Substring(0, projectSpec.Length-1) - let splitted = ``project spec``.Split([|':'; '/'|], StringSplitOptions.RemoveEmptyEntries) + let projectSpec = projectSpec.TrimEnd('/') + let ``project spec``, commit = + match projectSpec.IndexOf('/', 8) with // 8 = "https://".Length + | -1 -> projectSpec, "/" + | pos -> projectSpec.Substring(0, pos), projectSpec.Substring(pos) + let splitted = projectSpec.TrimEnd('/').Split([|':'; '/'|], StringSplitOptions.RemoveEmptyEntries) let fileName = match String.IsNullOrEmpty fileSpec with | true -> let name = Seq.last splitted if String.IsNullOrEmpty <| Path.GetExtension(name) then name + ".fs" else name | false -> fileSpec - match splitted |> Seq.truncate 4 |> Seq.toArray with + let owner = + match ``project spec``.IndexOf("://") with + | -1 -> ``project spec`` + | pos -> ``project spec``.Substring(pos+3) //SourceFile(origin(url), (owner,project, commit), path) - | [| protocol; domain |] -> HttpLink(``project spec``), (domain, domain, None), fileName - | [| protocol; domain; project |] -> HttpLink(``project spec``), (domain,project, None), fileName - | [| protocol; owner; project; details |] -> HttpLink(``project spec``), (owner,project+"/"+details, None), fileName - | _ -> failwithf "invalid http-reference specification:%s %s" Environment.NewLine trimmed + HttpLink(``project spec``), (owner, "", Some commit), fileName match parts with | [| _; projectSpec; |] -> getParts projectSpec String.Empty | [| _; projectSpec; fileSpec |] -> getParts projectSpec fileSpec diff --git a/src/Paket.Core/ModuleResolver.fs b/src/Paket.Core/ModuleResolver.fs index 358e1f9445..d433cc27fe 100644 --- a/src/Paket.Core/ModuleResolver.fs +++ b/src/Paket.Core/ModuleResolver.fs @@ -22,7 +22,7 @@ type UnresolvedSourceFile = override this.ToString() = let name = if this.Name = Constants.FullProjectSourceFileName then "" else " " + this.Name match this.Origin with - | HttpLink url -> sprintf "http %s%s" url name + | HttpLink url -> sprintf "http %s%s %s" url this.Commit.Value this.Name | _ -> let link = match this.Origin with @@ -54,10 +54,11 @@ type ResolvedSourceFile = let private getCommit (file : UnresolvedSourceFile) = defaultArg file.Commit "master" let resolve getDependencies getSha1 (file : UnresolvedSourceFile) : ResolvedSourceFile = - let sha = - file - |> getCommit - |> getSha1 file.Origin file.Owner file.Project + let sha = + let commit = getCommit file + match file.Origin with + | SingleSourceFileOrigin.HttpLink _ -> commit + | _ -> getSha1 file.Origin file.Owner file.Project commit let resolved = { Commit = sha @@ -79,10 +80,7 @@ let private detectConflicts (remoteFiles : UnresolvedSourceFile list) : unit = remoteFiles |> Seq.groupBy (fun file -> let directoryName = - let path = normalizePath (file.Name.TrimStart('/')) - match path.LastIndexOfAny([| '/'; '\\' |]) with - | -1 -> "" - | x -> path.Substring(0, x) + normalizePath (file.Name.TrimStart('/')) file.Owner, file.Project, directoryName) |> Seq.map (fun (key, files) -> key, files |> Seq.map getCommit |> Seq.distinct) |> Seq.filter (snd >> Seq.length >> (<) 1) diff --git a/src/Paket.Core/RemoteDownload.fs b/src/Paket.Core/RemoteDownload.fs index fb6fd6d371..4a698fbc24 100644 --- a/src/Paket.Core/RemoteDownload.fs +++ b/src/Paket.Core/RemoteDownload.fs @@ -115,7 +115,8 @@ let downloadRemoteFiles(remoteFile:ResolvedSourceFile,destination) = async { DirectoryCopy(source,projectPath,true) | SingleSourceFileOrigin.GistLink, _ -> return! downloadFromUrl(None,rawGistFileUrl remoteFile.Owner remoteFile.Project remoteFile.Name) destination | SingleSourceFileOrigin.GitHubLink, _ -> return! downloadFromUrl(None,rawFileUrl remoteFile.Owner remoteFile.Project remoteFile.Commit remoteFile.Name) destination - | SingleSourceFileOrigin.HttpLink(url), _ -> + | SingleSourceFileOrigin.HttpLink(origin), _ -> + let url = origin + remoteFile.Commit do! downloadFromUrl(None, url) destination match Path.GetExtension(destination).ToLowerInvariant() with | ".zip" -> diff --git a/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs b/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs index acc092a11f..672c085452 100644 --- a/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs +++ b/tests/Paket.Tests/DependenciesFile/ParserSpecs.fs @@ -278,15 +278,15 @@ let ``should read http source file from config without quotes with file specs``( dependencies.RemoteFiles |> shouldEqual [ { Owner = "www.fssnip.net" - Project = "raw/1M" + Project = "" Name = "test1.fs" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M" - Commit = None } + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net" + Commit = Some "/raw/1M" } { Owner = "www.fssnip.net" - Project = "raw/1M" + Project = "" Name = "src/test2.fs" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M/1" - Commit = None } ] + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net" + Commit = Some "/raw/1M/1" } ] [] let ``should read gist source file from config without quotes with file specs``() = @@ -335,30 +335,30 @@ let ``should read http source file from config without quotes, parsing rules``() dependencies.RemoteFiles |> shouldEqual [ { Owner = "example" - Project = "example" + Project = "" Name = "example.fs" Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example" - Commit = None } + Commit = Some "/" } { Owner = "example" - Project = "item" + Project = "" Name = "item.fs" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item" - Commit = None } + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example" + Commit = Some "/item" } { Owner = "example" - Project = "item" + Project = "" Name = "item.fs" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item" - Commit = None } + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example" + Commit = Some "/item" } { Owner = "example" - Project = "item/3" + Project = "" Name = "3.fs" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item/3" - Commit = None } + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example" + Commit = Some "/item/3" } { Owner = "example" - Project = "item/3" + Project = "" Name = "1.fs" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example/item/3/1" - Commit = None } ] + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://example" + Commit = Some "/item/3/1" } ] [] let ``should read http binary references from config``() = @@ -369,15 +369,15 @@ let ``should read http binary references from config``() = dependencies.RemoteFiles |> shouldEqual [ { Owner = "www.frijters.net" - Project = "ikvmbin-8.0.5449.0.zip" + Project = "" Name = "ikvmbin-8.0.5449.0.zip" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net/ikvmbin-8.0.5449.0.zip" - Commit = None } + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net" + Commit = Some "/ikvmbin-8.0.5449.0.zip" } { Owner = "www.frijters.net" - Project = "ikvmbin-8.0.5449.0.zip" + Project = "" Name = "ikvmbin.zip" - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net/ikvmbin-8.0.5449.0.zip" - Commit = None } ] + Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.frijters.net" + Commit = Some "/ikvmbin-8.0.5449.0.zip" } ] let configWithoutVersions = """ diff --git a/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs b/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs index 4e7acb89c2..9ed3670550 100644 --- a/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs +++ b/tests/Paket.Tests/Lockfile/GeneratorSpecs.fs @@ -130,10 +130,21 @@ let ``should generate other version ranges for packages``() = |> LockFileSerializer.serializePackages cfg.Options |> shouldEqual (normalizeLineEndings expected3) +let trivialResolve (f:ModuleResolver.UnresolvedSourceFile) = + { Commit = + match f.Commit with + | Some(v) -> v + | None -> "" + Owner = f.Owner + Origin = f.Origin + Project = f.Project + Dependencies = Set.empty + Name = f.Name } : ModuleResolver.ResolvedSourceFile + let expectedWithHttp = """HTTP - remote: http://www.fssnip.net/raw/1M + remote: http://www.fssnip.net specs: - test.fs""" + test.fs (/raw/1M)""" [] let ``should generate lock file for http source files``() = @@ -142,34 +153,23 @@ let ``should generate lock file for http source files``() = let cfg = DependenciesFile.FromCode(config) cfg.RemoteFiles - |> List.map (fun f -> - { Commit = "" - Owner = f.Owner - Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net/raw/1M" - Project = f.Project - Dependencies = Set.empty - Name = f.Name } : ModuleResolver.ResolvedSourceFile) + |> List.map trivialResolve |> LockFileSerializer.serializeSourceFiles |> shouldEqual (normalizeLineEndings expectedWithHttp) let expectedMultiple = """HTTP - remote: http://www.fssnip.net/raw/32 + remote: http://www.fssnip.net specs: - myFile2.fs + myFile.fs (/raw/1M) + myFile2.fs (/raw/32) + myFile3.fs (/raw/15) GIST remote: Thorium/1972308 specs: gistfile1.fs remote: Thorium/6088882 specs: - FULLPROJECT -HTTP - remote: http://www.fssnip.net/raw/1M - specs: - myFile.fs - remote: http://www.fssnip.net/raw/15 - specs: - myFile3.fs""" + FULLPROJECT""" [] let ``should generate lock file for http and gist source files``() = @@ -186,13 +186,35 @@ http http://www.fssnip.net/raw/15 myFile3.fs """ let cfg = DependenciesFile.FromCode(config) cfg.RemoteFiles - |> List.map (fun f -> - { Commit = "" - Owner = f.Owner - Origin = f.Origin - Project = f.Project - Dependencies = Set.empty - Name = f.Name } : ModuleResolver.ResolvedSourceFile) + |> List.map trivialResolve |> LockFileSerializer.serializeSourceFiles |> shouldEqual (normalizeLineEndings expectedMultiple) + +let expectedForStanfordNLPdotNET = """HTTP + remote: http://www.frijters.net + specs: + ikvmbin-8.0.5449.0.zip (/ikvmbin-8.0.5449.0.zip) + remote: http://nlp.stanford.edu + specs: + stanford-corenlp-full-2014-10-31.zip (/software/stanford-corenlp-full-2014-10-31.zip) + stanford-ner-2014-10-26.zip (/software/stanford-ner-2014-10-26.zip) + stanford-parser-full-2014-10-31.zip (/software/stanford-parser-full-2014-10-31.zip) + stanford-postagger-full-2014-10-26.zip (/software/stanford-postagger-full-2014-10-26.zip) + stanford-segmenter-2014-10-26.zip (/software/stanford-segmenter-2014-10-26.zip)""" + +[] +let ``should generate lock file for http Stanford.NLP.NET project``() = + let config = """http http://www.frijters.net/ikvmbin-8.0.5449.0.zip +http http://nlp.stanford.edu/software/stanford-corenlp-full-2014-10-31.zip +http http://nlp.stanford.edu/software/stanford-ner-2014-10-26.zip +http http://nlp.stanford.edu/software/stanford-parser-full-2014-10-31.zip +http http://nlp.stanford.edu/software/stanford-postagger-full-2014-10-26.zip +http http://nlp.stanford.edu/software/stanford-segmenter-2014-10-26.zip""" + + let cfg = DependenciesFile.FromCode(config) + + cfg.RemoteFiles + |> List.map trivialResolve + |> LockFileSerializer.serializeSourceFiles + |> shouldEqual (normalizeLineEndings expectedForStanfordNLPdotNET) \ No newline at end of file diff --git a/tests/Paket.Tests/Resolver/ConflictSourcesSpecs.fs b/tests/Paket.Tests/Resolver/ConflictSourcesSpecs.fs index 0318c90ec5..fd426aedda 100644 --- a/tests/Paket.Tests/Resolver/ConflictSourcesSpecs.fs +++ b/tests/Paket.Tests/Resolver/ConflictSourcesSpecs.fs @@ -42,25 +42,21 @@ let config2 = """ source "http://nuget.org/api/v2" github fsharp/fsharp:master foo.fs -github fsharp/fsharp:fsharp4 bar.fs +github fsharp/fsharp:fsharp4 foo.fs github fsprojects/FAKE:master test.fs github fsprojects/FAKE:vNext readme.md """ let expectedError = """Found conflicting source file requirements: - - fsharp/fsharp + - fsharp/fsharpfoo.fs Versions: - master - fsharp4 - - fsprojects/FAKE - Versions: - - master - - vNext Currently multiple versions for same source directory are not supported. Please adjust the dependencies file.""" |> normalizeLineEndings [] -let ``should fail resolving source files from same repository but different versions``() = +let ``should fail resolving same source files from same repository but different versions``() = try let cfg = DependenciesFile.FromCode(config2) ModuleResolver.Resolve(noGitHubConfigured, noGitHubConfigured, cfg.RemoteFiles) |> ignore