package runner import ( "bufio" "os" "path/filepath" "regexp" "slices" "strings" "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/katana/pkg/engine/headless/auth" "github.com/projectdiscovery/katana/pkg/types" "github.com/projectdiscovery/katana/pkg/utils" "github.com/projectdiscovery/utils/errkit" fileutil "github.com/projectdiscovery/utils/file" "gopkg.in/yaml.v3" ) // validateOptions validates the provided options for crawler func validateOptions(options *types.Options) error { if options.Interactive { if options.HeadlessHybrid { return errkit.New("interactive mode requires pure headless mode; -hh is not supported") } options.Headless = true if options.InteractionDepth < 1 || options.InteractionMaxStates < 0 || options.InteractionMaxActions < 0 || options.InteractionSettle < 0 || options.InteractionDuration < 0 { return errkit.New("invalid interactive limits: depth must be positive and other limits non-negative") } } if options.MaxDepth <= 0 && options.CrawlDuration.Seconds() <= 0 { return errkit.New("either max-depth or crawl-duration must be specified") } if len(options.URLs) == 0 && !fileutil.HasStdin() { return errkit.New("no inputs specified for crawler") } // Apply crawl strategy preset if specified if options.CrawlStrategy != "" { validPresets := []string{"fast", "balanced", "thorough"} strategyLower := strings.ToLower(strings.TrimSpace(options.CrawlStrategy)) if !slices.Contains(validPresets, strategyLower) { return errkit.New("invalid crawl-strategy: must be one of (fast, balanced, thorough)") } options.CrawlStrategy = strategyLower if !options.Headless && !options.HeadlessHybrid { options.Headless = true } applyCrawlStrategyPreset(options) } // Validate page load strategy if options.PageLoadStrategy != "" { validStrategies := []string{"heuristic", "load", "domcontentloaded", "networkidle", "none"} if !slices.Contains(validStrategies, options.PageLoadStrategy) { return errkit.New("invalid page-load-strategy: must be one of (heuristic, load, domcontentloaded, networkidle, none)") } } else { // Default to heuristic options.PageLoadStrategy = "heuristic" } // Disabling automatic form fill (-aff) for headless navigation due to incorrect implementation. // Form filling should be handled via headless actions within the page context if options.HeadlessHybrid && options.AutomaticFormFill { options.AutomaticFormFill = false gologger.Info().Msgf("Automatic form fill (-aff) has been disabled for headless navigation.") } // Disallow ambiguous engine selection if options.Headless && options.HeadlessHybrid { return errkit.New("flags -hl (headless) and -hh (hybrid) are mutually exclusive") } if options.AIReplay != "" && (options.AIURL != "" || options.AIRecord != "") { return errkit.New("-ai-replay cannot be combined with -ai-url or -ai-record") } if options.AIURL != "" && options.AIModel == "" { return errkit.New("-ai-url requires -ai-model") } if options.AIRecord != "" && options.AIURL == "" { return errkit.New("-ai-record requires -ai-url") } if options.MaxScrollSteps < 0 { return errkit.New("max-scroll-steps cannot be negative") } // Warn if -headless or -hh is used with -cwu (Chrome WebSocket URL) // The ChromeWSUrl takes precedence and pure headless engine will be used if options.Headless && options.ChromeWSUrl != "" { gologger.Warning().Msgf("Using -cwu with existing browser session. The -headless flag is redundant.") gologger.Info().Msgf("Connecting to Chrome at: %s", options.ChromeWSUrl) } else if options.HeadlessHybrid && options.ChromeWSUrl != "" { gologger.Warning().Msgf("Using -cwu forces pure headless engine. The -hh (hybrid) flag will be ignored.") gologger.Info().Msgf("Connecting to Chrome at: %s (using pure headless engine)", options.ChromeWSUrl) } else if options.ChromeWSUrl != "" { gologger.Info().Msgf("Connecting to Chrome at: %s (using pure headless engine)", options.ChromeWSUrl) } if options.AuthCredentials != "" { if !strings.Contains(options.AuthCredentials, ":") { return errkit.New("auth credentials must be in username:password format") } if !options.Headless && !options.HeadlessHybrid { options.Headless = true gologger.Info().Msgf("Headless mode enabled automatically for authenticated crawling.") } } if options.RecordedFlow != "" { if !fileutil.FileExists(options.RecordedFlow) { return errkit.New("recorded flow file does not exist") } // Recorded flows replay against the pure headless engine only. if options.HeadlessHybrid { options.HeadlessHybrid = false gologger.Warning().Msgf("Recorded flow (-rf) requires pure headless mode; disabling -hh.") } if !options.Headless { options.Headless = true gologger.Info().Msgf("Headless mode enabled automatically for recorded flow.") } username, password := "", "" if options.AuthCredentials != "" { parts := strings.SplitN(options.AuthCredentials, ":", 2) username = parts[0] if len(parts) > 1 { password = parts[1] } } steps, err := auth.StepsFromFile(options.RecordedFlow, username, password) if err != nil { return errkit.Wrap(err, "invalid recorded flow") } if auth.NeedsCredentials(steps) && options.AuthCredentials == "" { return errkit.New("recorded flow requires -auto-login username:password for credential placeholders") } replaySteps, loginURL := auth.StepsAfterFirstNavigateURL(steps) if loginURL == "" { return errkit.New("recorded flow requires an absolute http(s) navigate step") } if len(replaySteps) == 0 { return errkit.New("recorded flow requires at least one action after its navigate step") } } // Checked after -al and -rf, which enable pure headless mode themselves. if options.MaxScrollSteps > 0 && !options.Headless && options.ChromeWSUrl == "" { return errkit.New("headless (-hl) mode is required if -max-scroll-steps is set") } if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox || options.SystemChromePath != "" || options.UseInstalledChrome) && !options.Headless && !options.HeadlessHybrid { return errkit.New("headless (-hl) or hybrid (-hh) mode is required if -ho, -nos, -sc or -scp are set") } if options.SystemChromePath != "" { if !fileutil.FileExists(options.SystemChromePath) { return errkit.New("specified system chrome binary does not exist") } } if options.StoreResponseDir != "" && !options.StoreResponse { gologger.Debug().Msgf("store response directory specified, enabling \"sr\" flag automatically\n") options.StoreResponse = true } for _, mr := range options.OutputMatchRegex { cr, err := regexp.Compile(mr) if err != nil { return errkit.Wrap(err, "Invalid value for match regex option") } options.MatchRegex = append(options.MatchRegex, cr) } for _, fr := range options.OutputFilterRegex { cr, err := regexp.Compile(fr) if err != nil { return errkit.Wrap(err, "Invalid value for filter regex option") } options.FilterRegex = append(options.FilterRegex, cr) } if options.KnownFiles != "" && options.MaxDepth < 3 { gologger.Info().Msgf("Depth automatically set to 3 to accommodate the `--known-files` option (originally set to %d).", options.MaxDepth) options.MaxDepth = 3 } gologger.DefaultLogger.SetFormatter(formatter.NewCLI(options.NoColors)) return nil } // readCustomFormConfig reads custom form fill config func readCustomFormConfig(formConfig string) error { file, err := os.Open(formConfig) if err != nil { return errkit.Wrap(err, "could not read form config") } defer func() { if err := file.Close(); err != nil { gologger.Error().Msgf("Error closing file: %v\n", err) } }() var data utils.FormFillData if err := yaml.NewDecoder(file).Decode(&data); err != nil { return errkit.Wrap(err, "could not decode form config") } data.Resolve() utils.FormData = data return nil } // parseInputs parses the inputs returning a slice of URLs func (r *Runner) parseInputs() []string { values := make(map[string]struct{}) for _, url := range r.options.URLs { if url == "" { continue } value := normalizeInput(url) if _, ok := values[value]; !ok { values[value] = struct{}{} } } if r.stdin { scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { value := normalizeInput(scanner.Text()) if _, ok := values[value]; !ok { values[value] = struct{}{} } } } final := make([]string, 0, len(values)) for k := range values { final = append(final, k) } return final } func normalizeInput(value string) string { return strings.TrimSpace(value) } func initExampleFormFillConfig() error { homedir, err := os.UserHomeDir() if err != nil { return errkit.Wrap(err, "could not get home directory") } defaultConfig := filepath.Join(homedir, ".config", "katana", "form-config.yaml") if fileutil.FileExists(defaultConfig) { return readCustomFormConfig(defaultConfig) } if err := os.MkdirAll(filepath.Dir(defaultConfig), 0775); err != nil { return err } exampleConfig, err := os.Create(defaultConfig) if err != nil { return errkit.Wrap(err, "could not get home directory") } defer func() { if err := exampleConfig.Close(); err != nil { gologger.Error().Msgf("Error closing example config: %v\n", err) } }() err = yaml.NewEncoder(exampleConfig).Encode(utils.DefaultFormFillData) return err } // applyCrawlStrategyPreset configures headless parameters based on the selected preset. func applyCrawlStrategyPreset(options *types.Options) { switch options.CrawlStrategy { case "fast": if options.PageLoadStrategy == "" || options.PageLoadStrategy == "heuristic" { options.PageLoadStrategy = "domcontentloaded" } if options.DOMWaitTime == 0 || options.DOMWaitTime == 5 { options.DOMWaitTime = 1 } if options.InteractionDepth == 0 || options.InteractionDepth == 4 { options.InteractionDepth = 2 } if options.InteractionSettle == 0 || options.InteractionSettle == 2*time.Second { options.InteractionSettle = 500 * time.Millisecond } if options.MaxScrollSteps == 0 { options.MaxScrollSteps = 1 } if options.MaxOnclickLinks == 0 || options.MaxOnclickLinks == 10 { options.MaxOnclickLinks = 3 } case "balanced": if options.PageLoadStrategy == "" { options.PageLoadStrategy = "heuristic" } if options.DOMWaitTime == 0 || options.DOMWaitTime == 5 { options.DOMWaitTime = 3 } if options.InteractionDepth == 0 || options.InteractionDepth == 4 { options.InteractionDepth = 4 } if options.InteractionSettle == 0 || options.InteractionSettle == 2*time.Second { options.InteractionSettle = 1500 * time.Millisecond } if options.MaxScrollSteps == 0 { options.MaxScrollSteps = 2 } if options.MaxOnclickLinks == 0 || options.MaxOnclickLinks == 10 { options.MaxOnclickLinks = 10 } case "thorough": if options.PageLoadStrategy == "" || options.PageLoadStrategy == "heuristic" { options.PageLoadStrategy = "networkidle" } if options.DOMWaitTime == 0 || options.DOMWaitTime == 5 { options.DOMWaitTime = 5 } if options.InteractionDepth == 0 || options.InteractionDepth == 4 { options.InteractionDepth = 6 } if options.InteractionSettle == 0 || options.InteractionSettle == 2*time.Second { options.InteractionSettle = 3 * time.Second } if options.MaxScrollSteps == 0 { options.MaxScrollSteps = 5 } if options.MaxOnclickLinks == 0 || options.MaxOnclickLinks == 10 { options.MaxOnclickLinks = 25 } } }