// Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package dep import ( "bytes" "fmt" "io" "reflect" "regexp" "sort" "sync" "github.com/golang/dep/gps" "github.com/golang/dep/gps/pkgtree" "github.com/pelletier/go-toml" "github.com/pkg/errors" ) // ManifestName is the manifest file name used by dep. const ManifestName = "Gopkg.toml" // Errors var ( errInvalidConstraint = errors.Errorf("%q must be a TOML array of tables", "constraint") errInvalidOverride = errors.Errorf("%q must be a TOML array of tables", "override") errInvalidRequired = errors.Errorf("%q must be a TOML list of strings", "required") errInvalidIgnored = errors.Errorf("%q must be a TOML list of strings", "ignored") errInvalidNoVerify = errors.Errorf("%q must be a TOML list of strings", "noverify") errInvalidPrune = errors.Errorf("%q must be a TOML table of booleans", "prune") errInvalidPruneProject = errors.Errorf("%q must be a TOML array of tables", "prune.project") errInvalidMetadata = errors.New("metadata should be a TOML table") errInvalidProjectRoot = errors.New("ProjectRoot name validation failed") errInvalidPruneValue = errors.New("prune options values must be booleans") errPruneSubProject = errors.New("prune projects should not contain sub projects") errRootPruneContainsName = errors.Errorf("%q should not include a name", "prune") errInvalidRootPruneValue = errors.New("root prune options must be omitted instead of being set to false") errInvalidPruneProjectName = errors.Errorf("%q in %q must be a string", "name", "prune.project") errNoName = errors.New("no name provided") ) // Manifest holds manifest file data and implements gps.RootManifest. type Manifest struct { Constraints gps.ProjectConstraints Ovr gps.ProjectConstraints Ignored []string Required []string NoVerify []string PruneOptions gps.CascadingPruneOptions } type rawManifest struct { Constraints []rawProject `toml:"constraint,omitempty"` Overrides []rawProject `toml:"override,omitempty"` Ignored []string `toml:"ignored,omitempty"` Required []string `toml:"required,omitempty"` NoVerify []string `toml:"noverify,omitempty"` PruneOptions rawPruneOptions `toml:"prune,omitempty"` } type rawProject struct { Name string `toml:"name"` Branch string `toml:"branch,omitempty"` Revision string `toml:"revision,omitempty"` Version string `toml:"version,omitempty"` Source string `toml:"source,omitempty"` } type rawPruneOptions struct { UnusedPackages bool `toml:"unused-packages,omitempty"` NonGoFiles bool `toml:"non-go,omitempty"` GoTests bool `toml:"go-tests,omitempty"` //Projects []map[string]interface{} `toml:"project,omitempty"` Projects []map[string]interface{} } const ( pruneOptionUnusedPackages = "unused-packages" pruneOptionGoTests = "go-tests" pruneOptionNonGo = "non-go" ) // Constants representing per-project prune uint8 values. const ( pvnone uint8 = 0 // No per-project prune value was set in Gopkg.toml. pvtrue uint8 = 1 // Per-project prune value was explicitly set to true. pvfalse uint8 = 2 // Per-project prune value was explicitly set to false. ) // NewManifest instantites a new manifest. func NewManifest() *Manifest { return &Manifest{ Constraints: make(gps.ProjectConstraints), Ovr: make(gps.ProjectConstraints), PruneOptions: gps.CascadingPruneOptions{ DefaultOptions: gps.PruneNestedVendorDirs, PerProjectOptions: map[gps.ProjectRoot]gps.PruneOptionSet{}, }, } } func validateManifest(s string) ([]error, error) { var warns []error // Load the TomlTree from string tree, err := toml.Load(s) if err != nil { return warns, errors.Wrap(err, "unable to load TomlTree from string") } // Convert tree to a map manifest := tree.ToMap() // match abbreviated git hash (7chars) or hg hash (12chars) abbrevRevHash := regexp.MustCompile("^[a-f0-9]{7}([a-f0-9]{5})?$") // Look for unknown fields and collect errors for prop, val := range manifest { switch prop { case "metadata": // Check if metadata is of Map type if reflect.TypeOf(val).Kind() != reflect.Map { warns = append(warns, errInvalidMetadata) } case "constraint", "override": valid := true // Invalid if type assertion fails. Not a TOML array of tables. if rawProj, ok := val.([]interface{}); ok { // Check element type. Must be a map. Checking one element would be // enough because TOML doesn't allow mixing of types. if reflect.TypeOf(rawProj[0]).Kind() != reflect.Map { valid = false } if valid { // Iterate through each array of tables for _, v := range rawProj { ruleProvided := false props := v.(map[string]interface{}) // Check the individual field's key to be valid for key, value := range props { // Check if the key is valid switch key { case "name": case "branch", "version", "source": ruleProvided = true case "revision": ruleProvided = true if valueStr, ok := value.(string); ok { if abbrevRevHash.MatchString(valueStr) { warns = append(warns, fmt.Errorf("revision %q should not be in abbreviated form", valueStr)) } } case "metadata": // Check if metadata is of Map type if reflect.TypeOf(value).Kind() != reflect.Map { warns = append(warns, fmt.Errorf("metadata in %q should be a TOML table", prop)) } default: // unknown/invalid key warns = append(warns, fmt.Errorf("invalid key %q in %q", key, prop)) } } if _, ok := props["name"]; !ok { warns = append(warns, errNoName) } else if !ruleProvided && prop == "constraint" { warns = append(warns, fmt.Errorf("branch, version, revision, or source should be provided for %q", props["name"])) } } } } else { valid = false } if !valid { if prop == "constraint" { return warns, errInvalidConstraint } if prop == "override" { return warns, errInvalidOverride } } case "ignored", "required", "noverify": valid := true if rawList, ok := val.([]interface{}); ok { // Check element type of the array. TOML doesn't let mixing of types in // array. Checking one element would be enough. Empty array is valid. if len(rawList) > 0 && reflect.TypeOf(rawList[0]).Kind() != reflect.String { valid = false } } else { valid = false } if !valid { if prop == "ignored" { return warns, errInvalidIgnored } if prop == "required" { return warns, errInvalidRequired } if prop == "noverify" { return warns, errInvalidNoVerify } } case "prune": pruneWarns, err := validatePruneOptions(val, true) warns = append(warns, pruneWarns...) if err != nil { return warns, err } default: warns = append(warns, fmt.Errorf("unknown field in manifest: %v", prop)) } } return warns, nil } func validatePruneOptions(val interface{}, root bool) (warns []error, err error) { if reflect.TypeOf(val).Kind() != reflect.Map { return warns, errInvalidPrune } for key, value := range val.(map[string]interface{}) { switch key { case pruneOptionNonGo, pruneOptionGoTests, pruneOptionUnusedPackages: if option, ok := value.(bool); !ok { return warns, errInvalidPruneValue } else if root && !option { return warns, errInvalidRootPruneValue } case "name": if root { warns = append(warns, errRootPruneContainsName) } else if _, ok := value.(string); !ok { return warns, errInvalidPruneProjectName } case "project": if !root { return warns, errPruneSubProject } if reflect.TypeOf(value).Kind() != reflect.Slice { return warns, errInvalidPruneProject } for _, project := range value.([]interface{}) { projectWarns, err := validatePruneOptions(project, false) warns = append(warns, projectWarns...) if err != nil { return nil, err } } default: if root { warns = append(warns, errors.Errorf("unknown field %q in %q", key, "prune")) } else { warns = append(warns, errors.Errorf("unknown field %q in %q", key, "prune.project")) } } } return warns, err } func checkRedundantPruneOptions(co gps.CascadingPruneOptions) (warns []error) { for name, project := range co.PerProjectOptions { if project.UnusedPackages != pvnone { if (co.DefaultOptions&gps.PruneUnusedPackages != 0) == (project.UnusedPackages == pvtrue) { warns = append(warns, errors.Errorf("redundant prune option %q set for %q", pruneOptionUnusedPackages, name)) } } if project.NonGoFiles != pvnone { if (co.DefaultOptions&gps.PruneNonGoFiles != 0) == (project.NonGoFiles == pvtrue) { warns = append(warns, errors.Errorf("redundant prune option %q set for %q", pruneOptionNonGo, name)) } } if project.GoTests != pvnone { if (co.DefaultOptions&gps.PruneGoTestFiles != 0) == (project.GoTests == pvtrue) { warns = append(warns, errors.Errorf("redundant prune option %q set for %q", pruneOptionGoTests, name)) } } } return warns } // ValidateProjectRoots validates the project roots present in manifest. func ValidateProjectRoots(c *Ctx, m *Manifest, sm gps.SourceManager) error { // Channel to receive all the errors errorCh := make(chan error, len(m.Constraints)+len(m.Ovr)) var wg sync.WaitGroup validate := func(pr gps.ProjectRoot) { defer wg.Done() origPR, err := sm.DeduceProjectRoot(string(pr)) if err != nil { errorCh <- err } else if origPR != pr { errorCh <- fmt.Errorf("the name for %q should be changed to %q", pr, origPR) } } for pr := range m.Constraints { wg.Add(1) go validate(pr) } for pr := range m.Ovr { wg.Add(1) go validate(pr) } for pr := range m.PruneOptions.PerProjectOptions { wg.Add(1) go validate(pr) } wg.Wait() close(errorCh) var valErr error if len(errorCh) > 0 { valErr = errInvalidProjectRoot c.Err.Printf("The following issues were found in Gopkg.toml:\n\n") for err := range errorCh { c.Err.Println(" ✗", err.Error()) } c.Err.Println() } return valErr } // readManifest returns a Manifest read from r and a slice of validation warnings. func readManifest(r io.Reader) (*Manifest, []error, error) { buf := &bytes.Buffer{} _, err := buf.ReadFrom(r) if err != nil { return nil, nil, errors.Wrap(err, "unable to read byte stream") } warns, err := validateManifest(buf.String()) if err != nil { return nil, warns, errors.Wrap(err, "manifest validation failed") } raw := rawManifest{} err = toml.Unmarshal(buf.Bytes(), &raw) if err != nil { return nil, warns, errors.Wrap(err, "unable to parse the manifest as TOML") } m, err := fromRawManifest(raw, buf) if err != nil { return nil, warns, err } warns = append(warns, checkRedundantPruneOptions(m.PruneOptions)...) return m, warns, nil } func fromRawManifest(raw rawManifest, buf *bytes.Buffer) (*Manifest, error) { m := NewManifest() m.Constraints = make(gps.ProjectConstraints, len(raw.Constraints)) m.Ovr = make(gps.ProjectConstraints, len(raw.Overrides)) m.Ignored = raw.Ignored m.Required = raw.Required m.NoVerify = raw.NoVerify for i := 0; i < len(raw.Constraints); i++ { name, prj, err := toProject(raw.Constraints[i]) if err != nil { return nil, err } if _, exists := m.Constraints[name]; exists { return nil, errors.Errorf("multiple dependencies specified for %s, can only specify one", name) } m.Constraints[name] = prj } for i := 0; i < len(raw.Overrides); i++ { name, prj, err := toProject(raw.Overrides[i]) if err != nil { return nil, err } if _, exists := m.Ovr[name]; exists { return nil, errors.Errorf("multiple overrides specified for %s, can only specify one", name) } m.Ovr[name] = prj } // TODO(sdboyer) it is awful that we have to do this manual extraction tree, err := toml.Load(buf.String()) if err != nil { return nil, errors.Wrap(err, "unable to load TomlTree from string") } iprunemap := tree.Get("prune") if iprunemap == nil { return m, nil } // Previous validation already guaranteed that, if it exists, it's this map // type. m.PruneOptions = fromRawPruneOptions(iprunemap.(*toml.Tree).ToMap()) return m, nil } func fromRawPruneOptions(prunemap map[string]interface{}) gps.CascadingPruneOptions { opts := gps.CascadingPruneOptions{ DefaultOptions: gps.PruneNestedVendorDirs, PerProjectOptions: make(map[gps.ProjectRoot]gps.PruneOptionSet), } if val, has := prunemap[pruneOptionUnusedPackages]; has && val.(bool) { opts.DefaultOptions |= gps.PruneUnusedPackages } if val, has := prunemap[pruneOptionNonGo]; has && val.(bool) { opts.DefaultOptions |= gps.PruneNonGoFiles } if val, has := prunemap[pruneOptionGoTests]; has && val.(bool) { opts.DefaultOptions |= gps.PruneGoTestFiles } trinary := func(v interface{}) uint8 { b := v.(bool) if b { return pvtrue } return pvfalse } if projprunes, has := prunemap["project"]; has { for _, proj := range projprunes.([]interface{}) { var pr gps.ProjectRoot // This should be redundant, but being explicit doesn't hurt. pos := gps.PruneOptionSet{NestedVendor: pvtrue} for key, val := range proj.(map[string]interface{}) { switch key { case "name": pr = gps.ProjectRoot(val.(string)) case pruneOptionNonGo: pos.NonGoFiles = trinary(val) case pruneOptionGoTests: pos.GoTests = trinary(val) case pruneOptionUnusedPackages: pos.UnusedPackages = trinary(val) } } opts.PerProjectOptions[pr] = pos } } return opts } // toRawPruneOptions converts a gps.RootPruneOption's PruneOptions to rawPruneOptions // // Will panic if gps.RootPruneOption includes ProjectPruneOptions // See https://github.com/golang/dep/pull/1460#discussion_r158128740 for more information func toRawPruneOptions(co gps.CascadingPruneOptions) rawPruneOptions { if len(co.PerProjectOptions) != 0 { panic("toRawPruneOptions cannot convert ProjectOptions to rawPruneOptions") } raw := rawPruneOptions{} if (co.DefaultOptions & gps.PruneUnusedPackages) != 0 { raw.UnusedPackages = true } if (co.DefaultOptions & gps.PruneNonGoFiles) != 0 { raw.NonGoFiles = true } if (co.DefaultOptions & gps.PruneGoTestFiles) != 0 { raw.GoTests = true } return raw } // toProject interprets the string representations of project information held in // a rawProject, converting them into a proper gps.ProjectProperties. An // error is returned if the rawProject contains some invalid combination - // for example, if both a branch and version constraint are specified. func toProject(raw rawProject) (n gps.ProjectRoot, pp gps.ProjectProperties, err error) { n = gps.ProjectRoot(raw.Name) if raw.Branch != "" { if raw.Version != "" || raw.Revision != "" { return n, pp, errors.Errorf("multiple constraints specified for %s, can only specify one", n) } pp.Constraint = gps.NewBranch(raw.Branch) } else if raw.Version != "" { if raw.Revision != "" { return n, pp, errors.Errorf("multiple constraints specified for %s, can only specify one", n) } // always semver if we can pp.Constraint, err = gps.NewSemverConstraintIC(raw.Version) if err != nil { // but if not, fall back on plain versions pp.Constraint = gps.NewVersion(raw.Version) } } else if raw.Revision != "" { pp.Constraint = gps.Revision(raw.Revision) } else { // If the user specifies nothing, it means an open constraint (accept // anything). pp.Constraint = gps.Any() } pp.Source = raw.Source return n, pp, nil } // MarshalTOML serializes this manifest into TOML via an intermediate raw form. func (m *Manifest) MarshalTOML() ([]byte, error) { raw := m.toRaw() var buf bytes.Buffer enc := toml.NewEncoder(&buf).ArraysWithOneElementPerLine(true) err := enc.Encode(raw) return buf.Bytes(), errors.Wrap(err, "unable to marshal the lock to a TOML string") } // toRaw converts the manifest into a representation suitable to write to the manifest file func (m *Manifest) toRaw() rawManifest { raw := rawManifest{ Constraints: make([]rawProject, 0, len(m.Constraints)), Overrides: make([]rawProject, 0, len(m.Ovr)), Ignored: m.Ignored, Required: m.Required, NoVerify: m.NoVerify, } for n, prj := range m.Constraints { raw.Constraints = append(raw.Constraints, toRawProject(n, prj)) } sort.Sort(sortedRawProjects(raw.Constraints)) for n, prj := range m.Ovr { raw.Overrides = append(raw.Overrides, toRawProject(n, prj)) } sort.Sort(sortedRawProjects(raw.Overrides)) raw.PruneOptions = toRawPruneOptions(m.PruneOptions) return raw } type sortedRawProjects []rawProject func (s sortedRawProjects) Len() int { return len(s) } func (s sortedRawProjects) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s sortedRawProjects) Less(i, j int) bool { l, r := s[i], s[j] if l.Name < r.Name { return true } if r.Name < l.Name { return false } return l.Source < r.Source } func toRawProject(name gps.ProjectRoot, project gps.ProjectProperties) rawProject { raw := rawProject{ Name: string(name), Source: project.Source, } if v, ok := project.Constraint.(gps.Version); ok { switch v.Type() { case gps.IsRevision: raw.Revision = v.String() case gps.IsBranch: raw.Branch = v.String() case gps.IsSemver, gps.IsVersion: raw.Version = v.ImpliedCaretString() } return raw } // We simply don't allow for a case where the user could directly // express a 'none' constraint, so we can ignore it here. We also ignore // the 'any' case, because that's the other possibility, and it's what // we interpret not having any constraint expressions at all to mean. // if !gps.IsAny(pp.Constraint) && !gps.IsNone(pp.Constraint) { if !gps.IsAny(project.Constraint) && project.Constraint != nil { // Has to be a semver range. raw.Version = project.Constraint.ImpliedCaretString() } return raw } // DependencyConstraints returns a list of project-level constraints. func (m *Manifest) DependencyConstraints() gps.ProjectConstraints { return m.Constraints } // Overrides returns a list of project-level override constraints. func (m *Manifest) Overrides() gps.ProjectConstraints { return m.Ovr } // IgnoredPackages returns a set of import paths to ignore. func (m *Manifest) IgnoredPackages() *pkgtree.IgnoredRuleset { if m == nil { return pkgtree.NewIgnoredRuleset(nil) } return pkgtree.NewIgnoredRuleset(m.Ignored) } // HasConstraintsOn checks if the manifest contains either constraints or // overrides on the provided ProjectRoot. func (m *Manifest) HasConstraintsOn(root gps.ProjectRoot) bool { if _, has := m.Constraints[root]; has { return true } if _, has := m.Ovr[root]; has { return true } return false } // RequiredPackages returns a set of import paths to require. func (m *Manifest) RequiredPackages() map[string]bool { if m == nil || m == (*Manifest)(nil) { return map[string]bool{} } if len(m.Required) == 0 { return nil } mp := make(map[string]bool, len(m.Required)) for _, i := range m.Required { mp[i] = true } return mp }