-
Notifications
You must be signed in to change notification settings - Fork 886
/
Copy pathoptions.go
837 lines (775 loc) · 38.7 KB
/
options.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
package runner
import (
"fmt"
"math"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pkg/errors"
"golang.org/x/exp/maps"
"github.com/projectdiscovery/cdncheck"
"github.com/projectdiscovery/goconfig"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/httpx/common/customextract"
"github.com/projectdiscovery/httpx/common/customheader"
"github.com/projectdiscovery/httpx/common/customlist"
customport "github.com/projectdiscovery/httpx/common/customports"
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/stringz"
"github.com/projectdiscovery/networkpolicy"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
"github.com/projectdiscovery/utils/env"
fileutil "github.com/projectdiscovery/utils/file"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
updateutils "github.com/projectdiscovery/utils/update"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
)
const (
two = 2
defaultThreads = 50
DefaultResumeFile = "resume.cfg"
DefaultOutputDirectory = "output"
)
var (
PDCPApiKey = ""
TeamIDEnv = env.GetEnvOrDefault("PDCP_TEAM_ID", "")
)
// OnResultCallback (hostResult)
type OnResultCallback func(Result)
type ScanOptions struct {
Methods []string
StoreResponseDirectory string
RequestURI string
RequestBody string
VHost bool
OutputTitle bool
OutputCopyright bool
OutputStatusCode bool
OutputLocation bool
OutputContentLength bool
StoreResponse bool
OmitBody bool
OutputServerHeader bool
OutputWebSocket bool
OutputWithNoColor bool
OutputMethod bool
ResponseHeadersInStdout bool
ResponseInStdout bool
Base64ResponseInStdout bool
ChainInStdout bool
TLSProbe bool
CSPProbe bool
VHostInput bool
OutputContentType bool
Unsafe bool
Pipeline bool
HTTP2Probe bool
OutputIP bool
OutputCName bool
OutputCDN string
OutputResponseTime bool
PreferHTTPS bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
StoreChain bool
StoreVisionReconClusters bool
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
OutputExtractRegex string
extractRegexps map[string]*regexp.Regexp
ExcludeCDN bool
HostMaxErrors int
ProbeAllIPS bool
Favicon bool
LeaveDefaultPorts bool
OutputLinesCount bool
OutputWordsCount bool
Hashes string
Screenshot bool
UseInstalledChrome bool
DisableStdin bool
NoScreenshotBytes bool
NoHeadlessBody bool
ScreenshotTimeout time.Duration
ScreenshotIdle time.Duration
}
func (s *ScanOptions) Clone() *ScanOptions {
return &ScanOptions{
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
RequestBody: s.RequestBody,
VHost: s.VHost,
OutputTitle: s.OutputTitle,
OutputCopyright: s.OutputCopyright,
OutputStatusCode: s.OutputStatusCode,
OutputLocation: s.OutputLocation,
OutputContentLength: s.OutputContentLength,
StoreResponse: s.StoreResponse,
OmitBody: s.OmitBody,
OutputServerHeader: s.OutputServerHeader,
OutputWebSocket: s.OutputWebSocket,
OutputWithNoColor: s.OutputWithNoColor,
OutputMethod: s.OutputMethod,
ResponseHeadersInStdout: s.ResponseHeadersInStdout,
ResponseInStdout: s.ResponseInStdout,
Base64ResponseInStdout: s.Base64ResponseInStdout,
ChainInStdout: s.ChainInStdout,
TLSProbe: s.TLSProbe,
CSPProbe: s.CSPProbe,
OutputContentType: s.OutputContentType,
Unsafe: s.Unsafe,
Pipeline: s.Pipeline,
HTTP2Probe: s.HTTP2Probe,
OutputIP: s.OutputIP,
OutputCName: s.OutputCName,
OutputCDN: s.OutputCDN,
OutputResponseTime: s.OutputResponseTime,
PreferHTTPS: s.PreferHTTPS,
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
HostMaxErrors: s.HostMaxErrors,
Favicon: s.Favicon,
extractRegexps: s.extractRegexps,
LeaveDefaultPorts: s.LeaveDefaultPorts,
OutputLinesCount: s.OutputLinesCount,
OutputWordsCount: s.OutputWordsCount,
Hashes: s.Hashes,
Screenshot: s.Screenshot,
UseInstalledChrome: s.UseInstalledChrome,
NoScreenshotBytes: s.NoScreenshotBytes,
NoHeadlessBody: s.NoHeadlessBody,
ScreenshotTimeout: s.ScreenshotTimeout,
ScreenshotIdle: s.ScreenshotIdle,
}
}
// Options contains configuration options for httpx.
type Options struct {
CustomHeaders customheader.CustomHeaders
CustomPorts customport.CustomPorts
matchStatusCode []int
matchContentLength []int
filterStatusCode []int
filterContentLength []int
Output string
OutputAll bool
StoreResponseDir string
OmitBody bool
HTTPProxy string
SocksProxy string
InputFile string
InputTargetHost goflags.StringSlice
Methods string
RequestURI string
RequestURIs string
requestURIs []string
OutputMatchStatusCode string
OutputMatchContentLength string
OutputFilterStatusCode string
OutputFilterErrorPage bool
FilterOutDuplicates bool
OutputFilterContentLength string
InputRawRequest string
rawRequest string
RequestBody string
OutputFilterString goflags.StringSlice
OutputMatchString goflags.StringSlice
OutputFilterRegex goflags.StringSlice
OutputMatchRegex goflags.StringSlice
Retries int
Threads int
Timeout int
Delay time.Duration
filterRegexes []*regexp.Regexp
matchRegexes []*regexp.Regexp
VHost bool
VHostInput bool
Smuggling bool
ExtractTitle bool
ExtractCopyright bool
StatusCode bool
Location bool
ContentLength bool
FollowRedirects bool
RespectHSTS bool
StoreResponse bool
JSONOutput bool
CSVOutput bool
CSVOutputEncoding string
PdcpAuth string
PdcpAuthCredFile string
Silent bool
Version bool
Verbose bool
NoColor bool
OutputServerHeader bool
OutputWebSocket bool
ResponseHeadersInStdout bool
ResponseInStdout bool
Base64ResponseInStdout bool
ChainInStdout bool
FollowHostRedirects bool
MaxRedirects int
OutputMethod bool
TLSProbe bool
CSPProbe bool
OutputContentType bool
OutputIP bool
OutputCName bool
ExtractFqdn bool
Unsafe bool
Debug bool
DebugRequests bool
DebugResponse bool
Pipeline bool
HTTP2Probe bool
OutputCDN string
OutputResponseTime bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
TLSGrab bool
protocol string
ShowStatistics bool
StatsInterval int
RandomAgent bool
StoreChain bool
StoreVisionReconClusters bool
Deny customlist.CustomList
Allow customlist.CustomList
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
ResponseBodyPreviewSize int
OutputExtractRegexs goflags.StringSlice
OutputExtractPresets goflags.StringSlice
RateLimit int
RateLimitMinute int
Probe bool
Resume bool
resumeCfg *ResumeCfg
Exclude goflags.StringSlice
HostMaxErrors int
Stream bool
SkipDedupe bool
ProbeAllIPS bool
Resolvers goflags.StringSlice
Favicon bool
OutputFilterFavicon goflags.StringSlice
OutputMatchFavicon goflags.StringSlice
LeaveDefaultPorts bool
ZTLS bool
OutputLinesCount bool
OutputMatchLinesCount string
matchLinesCount []int
OutputFilterLinesCount string
Memprofile string
filterLinesCount []int
OutputWordsCount bool
OutputMatchWordsCount string
matchWordsCount []int
OutputFilterWordsCount string
filterWordsCount []int
Hashes string
Jarm bool
Asn bool
OutputMatchCdn goflags.StringSlice
OutputFilterCdn goflags.StringSlice
SniName string
OutputMatchResponseTime string
OutputFilterResponseTime string
HealthCheck bool
ListDSLVariable bool
OutputFilterCondition string
OutputMatchCondition string
StripFilter string
//The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err.
OnResult OnResultCallback
DisableUpdateCheck bool
NoDecode bool
Screenshot bool
UseInstalledChrome bool
TlsImpersonate bool
DisableStdin bool
HttpApiEndpoint string
NoScreenshotBytes bool
NoHeadlessBody bool
ScreenshotTimeout time.Duration
ScreenshotIdle time.Duration
// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
HeadlessOptionalArguments goflags.StringSlice
Protocol string
OutputFilterErrorPagePath string
DisableStdout bool
// AssetUpload
AssetUpload bool
// AssetName
AssetName string
// AssetID
AssetID string
// AssetFileUpload
AssetFileUpload string
TeamID string
// OnClose adds a callback function that is invoked when httpx is closed
// to be exact at end of existing closures
OnClose func()
Trace bool
// Optional pre-created objects to reduce allocations
Wappalyzer *wappalyzer.Wappalyze
Networkpolicy *networkpolicy.NetworkPolicy
CDNCheckClient *cdncheck.Client
}
// ParseOptions parses the command line options for application
func ParseOptions() *Options {
options := &Options{}
var cfgFile string
flagSet := goflags.NewFlagSet()
flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library.`)
flagSet.CreateGroup("input", "Input",
flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"),
flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"),
flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions),
)
flagSet.CreateGroup("Probes", "Probes",
flagSet.BoolVarP(&options.StatusCode, "status-code", "sc", false, "display response status-code"),
flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "display response content-length"),
flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "display response content-type"),
flagSet.BoolVar(&options.Location, "location", false, "display response redirect location"),
flagSet.BoolVar(&options.Favicon, "favicon", false, "display mmh3 hash for '/favicon.ico' file"),
flagSet.StringVar(&options.Hashes, "hash", "", "display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"),
flagSet.BoolVar(&options.Jarm, "jarm", false, "display jarm fingerprint hash"),
flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "display response time"),
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"),
flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"),
flagSet.BoolVar(&options.ExtractTitle, "title", false, "display page title"),
flagSet.BoolVar(&options.ExtractCopyright, "copyright", false, "display copyright year"),
flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"),
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"),
flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
flagSet.BoolVar(&options.OutputCName, "cname", false, "display host cname"),
flagSet.BoolVarP(&options.ExtractFqdn, "efqdn", "extract-fqdn", false, "get domain and subdomains from response body and header in jsonl/csv output"),
flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"),
flagSet.DynamicVar(&options.OutputCDN, "cdn", "true", "display cdn/waf in use"),
flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"),
)
flagSet.CreateGroup("headless", "Headless",
flagSet.BoolVarP(&options.Screenshot, "screenshot", "ss", false, "enable saving screenshot of the page using headless browser"),
flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "enable using local installed chrome for screenshot"),
flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVarP(&options.NoScreenshotBytes, "exclude-screenshot-bytes", "esb", false, "enable excluding screenshot bytes from json output"),
flagSet.BoolVarP(&options.NoHeadlessBody, "exclude-headless-body", "ehb", false, "enable excluding headless header from json output"),
flagSet.DurationVarP(&options.ScreenshotTimeout, "screenshot-timeout", "st", 10*time.Second, "set timeout for screenshot in seconds"),
flagSet.DurationVarP(&options.ScreenshotIdle, "screenshot-idle", "sid", 1*time.Second, "set idle time before taking screenshot in seconds"),
)
flagSet.CreateGroup("matchers", "Matchers",
flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"),
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"),
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"),
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"),
flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchString, "match-string", "ms", nil, "match response with specified string (-ms admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchRegex, "match-regex", "mr", nil, "match response with specified regex (-mr admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"),
flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"),
)
flagSet.CreateGroup("extractor", "Extractor",
flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "display response content with matched regex", goflags.StringSliceOptions),
flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, fmt.Sprintf("display response content matched by a pre-defined regex (%s)", strings.Join(maps.Keys(customextract.ExtractPresets), ",")), goflags.StringSliceOptions),
)
flagSet.CreateGroup("filters", "Filters",
flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"),
flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "filter response with ML based error page detection"),
flagSet.BoolVarP(&options.FilterOutDuplicates, "filter-duplicates", "fd", false, "filter out near-duplicate responses (only first response is retained)"),
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"),
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-ffc 1494302000)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterString, "filter-string", "fs", nil, "filter response with specified string (-fs admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterRegex, "filter-regex", "fe", nil, "filter response with specified regex (-fe admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"),
flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"),
flagSet.DynamicVar(&options.StripFilter, "strip", "html", "strips all tags in response. supported formats: html,xml"),
)
flagSet.CreateGroup("rate-limit", "Rate-Limit",
flagSet.IntVarP(&options.Threads, "threads", "t", defaultThreads, "number of threads to use"),
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum requests to send per second"),
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
)
flagSet.CreateGroup("Misc", "Miscellaneous",
flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "probe all the ips associated with same host"),
flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg http:1,2-10,11,https:80)"),
flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"),
flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "send http probes on the extracted TLS domains (dns_name)"),
flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "send http probes on the extracted CSP domains"),
flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "perform TLS(SSL) data grabbing"),
flagSet.BoolVar(&options.Pipeline, "pipeline", false, "probe and display server supporting HTTP1.1 pipeline"),
flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "probe and display server supporting HTTP2"),
flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"),
flagSet.BoolVarP(&options.ListDSLVariable, "list-dsl-variables", "ldv", false, "list json output field keys name that support dsl matcher/filter"),
)
flagSet.CreateGroup("update", "Update",
flagSet.CallbackVarP(GetUpdateCallback(), "update", "up", "update httpx to latest version"),
flagSet.BoolVarP(&options.DisableUpdateCheck, "disable-update-check", "duc", false, "disable automatic httpx update check"),
)
flagSet.CreateGroup("output", "Output",
flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"),
flagSet.BoolVarP(&options.OutputAll, "output-all", "oa", false, "filename to write output results in all formats"),
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"),
flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body in output"),
flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"),
flagSet.StringVarP(&options.CSVOutputEncoding, "csv-output-encoding", "csvo", "", "define output encoding"),
flagSet.BoolVarP(&options.JSONOutput, "json", "j", false, "store output in JSONL(ines) format"),
flagSet.BoolVarP(&options.ResponseHeadersInStdout, "include-response-header", "irh", false, "include http response (headers) in JSON output (-json only)"),
flagSet.BoolVarP(&options.ResponseInStdout, "include-response", "irr", false, "include http request/response (headers + body) in JSON output (-json only)"),
flagSet.BoolVarP(&options.Base64ResponseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"),
flagSet.BoolVar(&options.ChainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"),
flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"),
flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"),
flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11)"),
flagSet.StringVarP(&options.OutputFilterErrorPagePath, "filter-error-page-path", "fepp", "filtered_error_page.json", "path to store filtered error pages"),
)
flagSet.CreateGroup("configs", "Configurations",
flagSet.StringVar(&cfgFile, "config", "", "path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)"),
flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.NormalizedStringSliceOptions),
flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"),
flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"),
flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"),
flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"),
flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"),
flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"),
flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"),
flagSet.BoolVar(&options.Resume, "resume", false, "resume scan using resume.cfg"),
flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "follow http redirects"),
flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "maxr", 10, "max number of redirects to follow per host"),
flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
flagSet.BoolVarP(&options.RespectHSTS, "respect-hsts", "rhsts", false, "respect HSTS response headers for redirect requests"),
flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "get a list of vhosts as input"),
flagSet.StringVar(&options.Methods, "x", "", "request methods to probe, use 'all' to probe all HTTP methods"),
flagSet.StringVar(&options.RequestBody, "body", "", "post body to include in http request"),
flagSet.BoolVarP(&options.Stream, "stream", "s", false, "stream mode - start elaborating input targets without sorting"),
flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"),
flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https://host:443"),
flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"),
flagSet.BoolVar(&options.NoDecode, "no-decode", false, "avoid decoding body"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"),
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
)
flagSet.CreateGroup("debug", "Debug",
flagSet.BoolVarP(&options.HealthCheck, "hc", "health-check", false, "run diagnostic check up"),
flagSet.BoolVar(&options.Debug, "debug", false, "display request/response content in cli"),
flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "display request content in cli"),
flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "display response content in cli"),
flagSet.BoolVar(&options.Version, "version", false, "display httpx version"),
flagSet.BoolVar(&options.ShowStatistics, "stats", false, "display scan statistic"),
flagSet.StringVar(&options.Memprofile, "profile-mem", "", "optional httpx memory profile dump file"),
flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"),
flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"),
flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"),
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"),
flagSet.BoolVarP(&options.Trace, "trace", "tr", false, "trace"),
)
flagSet.CreateGroup("Optimizations", "Optimizations",
flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "display both probed protocol (HTTPS and HTTP)"),
flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "),
flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"),
flagSet.StringSliceVarP(&options.Exclude, "exclude", "e", nil, "exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)", goflags.CommaSeparatedStringSliceOptions),
flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"),
flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"),
flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"),
flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"),
flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"),
)
flagSet.CreateGroup("cloud", "Cloud",
flagSet.DynamicVar(&options.PdcpAuth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
flagSet.StringVarP(&options.PdcpAuthCredFile, "auth-config", "ac", "", "configure projectdiscovery cloud (pdcp) api key credential file"),
flagSet.BoolVarP(&options.AssetUpload, "dashboard", "pd", false, "upload / view output in projectdiscovery cloud (pdcp) UI dashboard"),
flagSet.StringVarP(&options.TeamID, "team-id", "tid", TeamIDEnv, "upload asset results to given team id (optional)"),
flagSet.StringVarP(&options.AssetID, "asset-id", "aid", "", "upload new assets to existing asset id (optional)"),
flagSet.StringVarP(&options.AssetName, "asset-name", "aname", "", "assets group name to set (optional)"),
flagSet.StringVarP(&options.AssetFileUpload, "dashboard-upload", "pdu", "", "upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard"),
)
_ = flagSet.Parse()
if options.OutputAll && options.Output == "" {
gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all")
}
if options.OutputAll {
options.JSONOutput = true
options.CSVOutput = true
}
if cfgFile != "" {
if !fileutil.FileExists(cfgFile) {
gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile)
}
// merge config file with flags
if err := flagSet.MergeConfigFile(cfgFile); err != nil {
gologger.Fatal().Msgf("Could not read config: %s\n", err)
}
}
if options.PdcpAuthCredFile != "" {
pdcpauth.PDCPCredFile = options.PdcpAuthCredFile
pdcpauth.PDCPDir = filepath.Dir(pdcpauth.PDCPCredFile)
}
// api key hierarchy: cli flag > env var > .pdcp/credential file
if options.PdcpAuth == "true" {
AuthWithPDCP()
} else if len(options.PdcpAuth) == 36 {
PDCPApiKey = options.PdcpAuth
ph := pdcpauth.PDCPCredHandler{}
if _, err := ph.GetCreds(); err == pdcpauth.ErrNoCreds {
apiServer := env.GetEnvOrDefault("PDCP_API_SERVER", pdcpauth.DefaultApiServer)
if validatedCreds, err := ph.ValidateAPIKey(PDCPApiKey, apiServer, "httpx"); err == nil {
_ = ph.SaveCreds(validatedCreds)
}
}
}
if options.HealthCheck {
gologger.Print().Msgf("%s\n", DoHealthCheck(options, flagSet))
os.Exit(0)
}
if options.StatsInterval != 0 {
options.ShowStatistics = true
}
if options.ResponseBodyPreviewSize > 0 && options.StripFilter == "" {
options.StripFilter = "html"
}
// Read the inputs and configure the logging
options.configureOutput()
err := options.configureResume()
if err != nil {
gologger.Fatal().Msgf("%s\n", err)
}
if options.ListDSLVariable {
dslVars, err := dslVariables()
if err != nil {
gologger.Fatal().Msgf("%s\n", err)
}
for _, dsl := range dslVars {
gologger.Print().Msg(dsl)
}
os.Exit(0)
}
showBanner()
if options.Version {
gologger.Info().Msgf("Current Version: %s\n", Version)
os.Exit(0)
}
if !options.DisableUpdateCheck {
latestVersion, err := updateutils.GetToolVersionCallback("httpx", Version)()
if err != nil {
if options.Verbose {
gologger.Error().Msgf("httpx version check failed: %v", err.Error())
}
} else {
gologger.Info().Msgf("Current httpx version %v %v", Version, updateutils.GetVersionDescription(Version, latestVersion))
}
}
if err := options.ValidateOptions(); err != nil {
gologger.Fatal().Msgf("%s\n", err)
}
return options
}
func (options *Options) ValidateOptions() error {
if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
return fmt.Errorf("file '%s' does not exist", options.InputFile)
}
if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) {
return fmt.Errorf("file '%s' does not exist", options.InputRawRequest)
}
if options.Silent {
incompatibleFlagsList := flagsIncompatibleWithSilent(options)
if len(incompatibleFlagsList) > 0 {
last := incompatibleFlagsList[len(incompatibleFlagsList)-1]
first := incompatibleFlagsList[:len(incompatibleFlagsList)-1]
msg := ""
if len(incompatibleFlagsList) > 1 {
msg += fmt.Sprintf("%s and %s flags are", strings.Join(first, ", "), last)
} else {
msg += fmt.Sprintf("%s flag is", last)
}
msg += " incompatible with silent flag"
return errors.New(msg)
}
}
var err error
if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil {
return errors.Wrap(err, "Invalid value for match status code option")
}
if options.matchContentLength, err = stringz.StringToSliceInt(options.OutputMatchContentLength); err != nil {
return errors.Wrap(err, "Invalid value for match content length option")
}
if options.filterStatusCode, err = stringz.StringToSliceInt(options.OutputFilterStatusCode); err != nil {
return errors.Wrap(err, "Invalid value for filter status code option")
}
if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil {
return errors.Wrap(err, "Invalid value for filter content length option")
}
for _, filterRegexStr := range options.OutputFilterRegex {
filterRegex, err := regexp.Compile(filterRegexStr)
if err != nil {
return errors.Wrap(err, "Invalid value for regex filter option")
}
options.filterRegexes = append(options.filterRegexes, filterRegex)
}
for _, matchRegexStr := range options.OutputMatchRegex {
matchRegex, err := regexp.Compile(matchRegexStr)
if err != nil {
return errors.Wrap(err, "Invalid value for match regex option")
}
options.matchRegexes = append(options.matchRegexes, matchRegex)
}
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
return errors.Wrap(err, "Invalid value for match lines count option")
}
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
return errors.Wrap(err, "Invalid value for match words count option")
}
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
return errors.Wrap(err, "Invalid value for filter lines count option")
}
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
return errors.Wrap(err, "Invalid value for filter words count option")
}
var resolvers []string
for _, resolver := range options.Resolvers {
if fileutil.FileExists(resolver) {
chFile, err := fileutil.ReadFile(resolver)
if err != nil {
return errors.Wrapf(err, "Couldn't process resolver file \"%s\"", resolver)
}
for line := range chFile {
resolvers = append(resolvers, line)
}
} else {
resolvers = append(resolvers, resolver)
}
}
options.Resolvers = resolvers
if len(options.Resolvers) > 0 {
gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ","))
}
if options.Screenshot && !options.StoreResponse {
gologger.Debug().Msgf("automatically enabling store response")
options.StoreResponse = true
}
if options.StoreResponse && options.StoreResponseDir == "" {
gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory)
options.StoreResponseDir = DefaultOutputDirectory
}
if options.StoreResponseDir != "" && !options.StoreResponse {
gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
options.StoreResponse = true
}
if options.Hashes != "" {
for _, hashType := range strings.Split(options.Hashes, ",") {
if !sliceutil.Contains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) {
gologger.Error().Msgf("Unsupported hash type: %s\n", hashType)
}
}
}
if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 {
options.OutputCDN = "true"
}
if !stringsutil.EqualFoldAny(options.Protocol, string(httpx.UNKNOWN), string(httpx.HTTP11)) {
return fmt.Errorf("invalid protocol: %s", options.Protocol)
}
if options.Threads == 0 {
gologger.Info().Msgf("Threads automatically set to %d", defaultThreads)
options.Threads = defaultThreads
}
return nil
}
// redundant with katana
func (options *Options) ParseHeadlessOptionalArguments() map[string]string {
var (
lastKey string
optionalArguments = make(map[string]string)
)
for _, v := range options.HeadlessOptionalArguments {
if v == "" {
continue
}
if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 {
key := strings.TrimSpace(argParts[0])
value := strings.TrimSpace(argParts[1])
if key != "" && value != "" {
optionalArguments[key] = value
lastKey = key
}
} else if !strings.HasPrefix(v, "--") {
optionalArguments[lastKey] += "," + v
} else {
optionalArguments[v] = ""
}
}
return optionalArguments
}
// configureOutput configures the output on the screen
func (options *Options) configureOutput() {
// If the user desires verbose output, show verbose output
if options.Verbose {
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
}
if options.Debug {
gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
}
if options.NoColor {
gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
}
if options.Silent {
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
}
if len(options.OutputMatchResponseTime) > 0 || len(options.OutputFilterResponseTime) > 0 {
options.OutputResponseTime = true
}
if options.CSVOutputEncoding != "" {
options.CSVOutput = true
}
}
func (options *Options) configureResume() error {
options.resumeCfg = &ResumeCfg{}
if options.Resume && fileutil.FileExists(DefaultResumeFile) {
return goconfig.Load(&options.resumeCfg, DefaultResumeFile)
}
return nil
}
// ShouldLoadResume resume file
func (options *Options) ShouldLoadResume() bool {
return options.Resume && fileutil.FileExists(DefaultResumeFile)
}
// ShouldSaveResume file
func (options *Options) ShouldSaveResume() bool {
return true
}
func flagsIncompatibleWithSilent(options *Options) []string {
var incompatibleFlagsList []string
for k, v := range map[string]bool{
"debug": options.Debug,
"debug-request": options.DebugRequests,
"debug-response": options.DebugResponse,
"verbose": options.Verbose,
} {
if v {
incompatibleFlagsList = append(incompatibleFlagsList, k)
}
}
return incompatibleFlagsList
}