From 6a53fb397ef410001e6b10479275373dab5a93c3 Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:36:13 +0530 Subject: [PATCH] chore(providers): remove the GPL holehe provider holehe is GPL-3.0 and the browser service is to become closed source. Remove the /v1/holehe route, models, scraper, dependency (and the packages only it pulled in), the Go provider, browserclient method and types, defaults, provenance, catalog, registry and contract fixtures, and the live docs. A stored workbook chain that still names holehe is now skipped as unknown_provider instead of failing the run or stopping the waterfall. Signed-off-by: Shimul Debnath <270455167+velixio@users.noreply.github.com> --- .../internal/browserclient/client.go | 7 - .../internal/browserclient/contract_test.go | 2 - .../internal/browserclient/recorded_test.go | 16 +- .../browserclient/testdata/openapi.json | 247 ------------------ .../testdata/recorded/holehe.json | 33 --- .../testdata/recorded/stats.json | 5 - .../internal/browserclient/types.go | 39 --- .../internal/providers/scrape/library.go | 44 ---- .../internal/providers/scrape/register.go | 1 - .../internal/providers/scrape/scrape.go | 3 +- .../providers/scrape/testdata/vectors.json | 115 -------- .../internal/providers/scrape/unit_test.go | 9 +- .../internal/providers/scrape/vectors_test.go | 9 - .../testdata/python_provider_inventory.json | 30 --- .../internal/vendorcatalog/native_costs.go | 1 - .../testdata/catalog_vectors.json | 23 -- .../internal/wbregistry/registry.json | 20 +- apps/backend-go/internal/workbook/cell.go | 9 + apps/backend-go/internal/workbook/defaults.go | 4 +- .../internal/workbook/provenance.go | 2 +- .../removed_provider_integration_test.go | 52 ++++ apps/backend-go/internal/workbook/runner.go | 10 + .../workbook/testdata/wbexec_vectors.json | 1 - .../testdata/contract/meta/providers.json | 17 -- .../contract/meta/providers_no_ws_header.json | 17 -- apps/browser-service/pyproject.toml | 1 - .../scripts/record_contract_fixtures.py | 18 +- .../src/browser_service/app.py | 8 +- .../src/browser_service/models.py | 27 -- .../browser_service/scrapers/holehe_check.py | 116 -------- apps/browser-service/tests/test_ops.py | 88 +------ apps/browser-service/uv.lock | 105 -------- .../src/content/docs/concepts/waterfalls.md | 2 +- .../docs/src/content/docs/guides/providers.md | 1 - docs/go-port/runbooks/browser-service.md | 3 +- 35 files changed, 86 insertions(+), 999 deletions(-) delete mode 100644 apps/backend-go/internal/browserclient/testdata/recorded/holehe.json create mode 100644 apps/backend-go/internal/workbook/removed_provider_integration_test.go delete mode 100644 apps/browser-service/src/browser_service/scrapers/holehe_check.py diff --git a/apps/backend-go/internal/browserclient/client.go b/apps/backend-go/internal/browserclient/client.go index 4b85a87ed..2e439efca 100644 --- a/apps/backend-go/internal/browserclient/client.go +++ b/apps/backend-go/internal/browserclient/client.go @@ -296,13 +296,6 @@ func (c *Client) Extract(ctx context.Context, req ExtractRequest) (ExtractResult return out, c.post(ctx, "/v1/extract", req, req.Timeout, &out) } -// Holehe checks which sites an email is registered on. -func (c *Client) Holehe(ctx context.Context, req HoleheRequest) (HoleheResult, error) { - req.Timeout = clamp(req.Timeout, 60*time.Second) - var out HoleheResult - return out, c.post(ctx, "/v1/holehe", req, req.Timeout, &out) -} - // StaffSpy scrapes a company's LinkedIn roster. func (c *Client) StaffSpy(ctx context.Context, req StaffSpyRequest) (StaffSpyResult, error) { req.Timeout = clamp(req.Timeout, 180*time.Second) diff --git a/apps/backend-go/internal/browserclient/contract_test.go b/apps/backend-go/internal/browserclient/contract_test.go index cc77efecf..076b17fed 100644 --- a/apps/backend-go/internal/browserclient/contract_test.go +++ b/apps/backend-go/internal/browserclient/contract_test.go @@ -323,8 +323,6 @@ func endpoints() []endpoint { {"post", "/v1/extract", ExtractRequest{HTML: "

x

", URL: "https://e.com/a", Engine: "browser", IncludeMarkdown: true, MaxBytes: 4096, UpstreamProxy: "http://1.2.3.4:8080", Timeout: 20 * time.Second}, "ExtractRequest", &ExtractResult{}, "ExtractResult"}, - {"post", "/v1/holehe", HoleheRequest{Email: "a@b.example", MaxModules: 10, UpstreamProxy: "http://1.2.3.4:8080", - Timeout: 30 * time.Second}, "HoleheRequest", &HoleheResult{}, "HoleheResult"}, {"post", "/v1/staffspy", StaffSpyRequest{CompanyName: "Acme", SearchTerm: "ceo", Location: "Pune", MaxResults: 5, Cookies: li, UpstreamProxy: "http://1.2.3.4:8080", Timeout: 60 * time.Second}, "StaffSpyRequest", &StaffSpyResult{}, "StaffSpyResult"}, diff --git a/apps/backend-go/internal/browserclient/recorded_test.go b/apps/backend-go/internal/browserclient/recorded_test.go index bb5a174c2..e086be5a1 100644 --- a/apps/backend-go/internal/browserclient/recorded_test.go +++ b/apps/backend-go/internal/browserclient/recorded_test.go @@ -121,21 +121,9 @@ func TestRecordedExtract(t *testing.T) { } } -func TestRecordedHoleheStaffSpyStats(t *testing.T) { - f := loadFixture(t, "holehe") +func TestRecordedStaffSpyStats(t *testing.T) { + f := loadFixture(t, "staffspy") c, sent := replay(t, f) - h, err := c.Holehe(context.Background(), HoleheRequest{Email: "jane@acme.example"}) - if err != nil { - t.Fatal(err) - } - sameRequest(t, "holehe", *sent, f.Request) - strict(t, "holehe", f.Response, &HoleheResult{}) - if !reflect.DeepEqual(h.ExistsOn, []string{"twitter"}) || h.TotalChecked != 2 || !h.Sites[1].RateLimited { - t.Fatalf("%+v", h) - } - - f = loadFixture(t, "staffspy") - c, sent = replay(t, f) s, err := c.StaffSpy(context.Background(), StaffSpyRequest{CompanyName: "Acme", Cookies: []Cookie{{Name: "li_at", Value: "x", Domain: ".linkedin.com"}}}) if err != nil { diff --git a/apps/backend-go/internal/browserclient/testdata/openapi.json b/apps/backend-go/internal/browserclient/testdata/openapi.json index d49e3fe42..787b5a2ba 100644 --- a/apps/backend-go/internal/browserclient/testdata/openapi.json +++ b/apps/backend-go/internal/browserclient/testdata/openapi.json @@ -624,135 +624,6 @@ "title": "Health", "type": "object" }, - "HoleheRequest": { - "additionalProperties": false, - "properties": { - "email": { - "maxLength": 254, - "minLength": 3, - "pattern": "^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", - "title": "Email", - "type": "string" - }, - "max_modules": { - "default": 50, - "description": "legacy checked the first 50 modules", - "maximum": 200.0, - "minimum": 1.0, - "title": "Max Modules", - "type": "integer" - }, - "timeout_s": { - "default": 60.0, - "maximum": 300.0, - "minimum": 5.0, - "title": "Timeout S", - "type": "number" - }, - "upstream_proxy": { - "anyOf": [ - { - "maxLength": 2048, - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "description": "upstream proxy (http/socks5/socks4 URL) the caller chose for this request; egress still goes through the service's guard first. Never logged or stored.", - "title": "Upstream Proxy" - } - }, - "required": [ - "email" - ], - "title": "HoleheRequest", - "type": "object" - }, - "HoleheResult": { - "properties": { - "elapsed_ms": { - "title": "Elapsed Ms", - "type": "integer" - }, - "email": { - "title": "Email", - "type": "string" - }, - "errors": { - "description": "modules that raised", - "title": "Errors", - "type": "integer" - }, - "exists_on": { - "items": { - "type": "string" - }, - "title": "Exists On", - "type": "array" - }, - "partial": { - "title": "Partial", - "type": "boolean" - }, - "sites": { - "items": { - "$ref": "#/components/schemas/HoleheSite" - }, - "title": "Sites", - "type": "array" - }, - "total_checked": { - "title": "Total Checked", - "type": "integer" - } - }, - "required": [ - "email", - "sites", - "exists_on", - "total_checked", - "errors", - "partial", - "elapsed_ms" - ], - "title": "HoleheResult", - "type": "object" - }, - "HoleheSite": { - "properties": { - "domain": { - "title": "Domain", - "type": "string" - }, - "exists": { - "title": "Exists", - "type": "boolean" - }, - "method": { - "title": "Method", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "rate_limited": { - "title": "Rate Limited", - "type": "boolean" - } - }, - "required": [ - "name", - "domain", - "method", - "exists", - "rate_limited" - ], - "title": "HoleheSite", - "type": "object" - }, "JobBoardsRequest": { "additionalProperties": false, "properties": { @@ -2338,124 +2209,6 @@ "summary": "Health" } }, - "/v1/holehe": { - "post": { - "operationId": "holehe_v1_holehe_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HoleheRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HoleheResult" - } - } - }, - "description": "Successful Response" - }, - "400": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Bad Request" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Unauthorized" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Forbidden" - }, - "413": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Content Too Large" - }, - "429": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Too Many Requests" - }, - "500": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Internal Server Error" - }, - "502": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Bad Gateway" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Service Unavailable" - }, - "504": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Gateway Timeout" - } - }, - "summary": "Holehe" - } - }, "/v1/render": { "post": { "operationId": "render_v1_render_post", diff --git a/apps/backend-go/internal/browserclient/testdata/recorded/holehe.json b/apps/backend-go/internal/browserclient/testdata/recorded/holehe.json deleted file mode 100644 index 87e2dba33..000000000 --- a/apps/backend-go/internal/browserclient/testdata/recorded/holehe.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "method": "POST", - "path": "/v1/holehe", - "request": { - "email": "jane@acme.example" - }, - "response": { - "email": "jane@acme.example", - "errors": 0, - "exists_on": [ - "twitter" - ], - "partial": false, - "sites": [ - { - "domain": "twitter.com", - "exists": true, - "method": "register", - "name": "twitter", - "rate_limited": false - }, - { - "domain": "spotify.com", - "exists": false, - "method": "register", - "name": "spotify", - "rate_limited": true - } - ], - "total_checked": 2 - }, - "status": 200 -} diff --git a/apps/backend-go/internal/browserclient/testdata/recorded/stats.json b/apps/backend-go/internal/browserclient/testdata/recorded/stats.json index eae106b7f..6d689b32b 100644 --- a/apps/backend-go/internal/browserclient/testdata/recorded/stats.json +++ b/apps/backend-go/internal/browserclient/testdata/recorded/stats.json @@ -16,11 +16,6 @@ "errors": {}, "ok": 2 }, - "holehe": { - "calls": 1, - "errors": {}, - "ok": 1 - }, "search": { "calls": 2, "errors": {}, diff --git a/apps/backend-go/internal/browserclient/types.go b/apps/backend-go/internal/browserclient/types.go index a9f8bc2bd..d3a265f82 100644 --- a/apps/backend-go/internal/browserclient/types.go +++ b/apps/backend-go/internal/browserclient/types.go @@ -420,45 +420,6 @@ type ExtractResult struct { ElapsedMS int `json:"elapsed_ms"` } -// ── holehe ────────────────────────────────────────────────────────── - -// HoleheRequest is POST /v1/holehe. MaxModules zero means 50 (legacy). -// Timeout zero means 60 s. -type HoleheRequest struct { - Email string `json:"email"` - MaxModules int `json:"max_modules,omitempty"` - UpstreamProxy ProxyURL `json:"upstream_proxy,omitempty"` - Timeout time.Duration `json:"-"` -} - -func (r HoleheRequest) MarshalJSON() ([]byte, error) { - type alias HoleheRequest - return json.Marshal(struct { - alias - TimeoutS float64 `json:"timeout_s,omitempty"` - }{alias(r), secs(r.Timeout)}) -} - -type HoleheSite struct { - Name string `json:"name"` - Domain string `json:"domain"` - Method string `json:"method"` - Exists bool `json:"exists"` - RateLimited bool `json:"rate_limited"` -} - -// HoleheResult: raw per-site results in module order. The verdict mapping -// (legacy is_real = len(ExistsOn) >= 2, email_verify, confidence) is Go's. -type HoleheResult struct { - Email string `json:"email"` - Sites []HoleheSite `json:"sites"` - ExistsOn []string `json:"exists_on"` - TotalChecked int `json:"total_checked"` - Errors int `json:"errors"` - Partial bool `json:"partial"` - ElapsedMS int `json:"elapsed_ms"` -} - // ── staffspy ──────────────────────────────────────────────────────── // StaffSpyRequest is POST /v1/staffspy (503 engine_unavailable unless the diff --git a/apps/backend-go/internal/providers/scrape/library.go b/apps/backend-go/internal/providers/scrape/library.go index e26e8293e..2b36e1088 100644 --- a/apps/backend-go/internal/providers/scrape/library.go +++ b/apps/backend-go/internal/providers/scrape/library.go @@ -16,50 +16,6 @@ import ( "github.com/debpalash/OpenGTM/apps/backend-go/internal/providers" ) -// ── holehe_verify.py (the holehe library runs as the browser op) ─────────── - -type holehe struct{ gated } - -func (holehe) Name() string { return "holehe" } - -func (holehe) Enrich(ctx context.Context, _ *http.Client, _ func() (string, error), in providers.Input) (providers.Output, error) { - email := ls(in.Lead, "email") - if email == "" || !strings.Contains(email, "@") { - return fail(in, "No email to verify"), nil - } - b := current().Browser - if b == nil { - return fail(in, errNoWeb), nil - } - res, err := b.Holehe(ctx, browserclient.HoleheRequest{Email: email}) - if err != nil { - if cerr := ctx.Err(); cerr != nil { - return providers.Output{}, cerr - } - // Deviation: Python reported the library exception text; the browser - // service returns fixed error codes only. - return fail(in, "holehe unavailable: "+browserCode(err)), nil - } - exists := res.ExistsOn - f := providers.NewObj() - f.Set("email", email) - var conf float64 - switch { - case len(exists) >= 2: - f.Set("email_verify", "verified_active") - f.Set("email_presence", strings.Join(exists[:min(5, len(exists))], ", ")) - conf = math.Min(0.95, 0.70+float64(len(exists))*0.05) - case len(exists) == 1: - f.Set("email_verify", "likely_valid") - f.Set("email_presence", strings.Join(exists, ", ")) - conf = 0.65 - default: - f.Set("email_verify", "no_presence") - conf = 0.30 - } - return result(in, f, conf, ""), nil -} - func browserCode(err error) string { var be *browserclient.Error if errors.As(err, &be) && be.Code != "" { diff --git a/apps/backend-go/internal/providers/scrape/register.go b/apps/backend-go/internal/providers/scrape/register.go index bde951794..20c36dba1 100644 --- a/apps/backend-go/internal/providers/scrape/register.go +++ b/apps/backend-go/internal/providers/scrape/register.go @@ -15,7 +15,6 @@ func init() { providers.Register(ddgEmail{}) providers.Register(decisionMaker{}) providers.Register(JobSpy{}) - providers.Register(holehe{}) providers.Register(staffSpy{}) providers.Register(mailScout{}) providers.Register(companySizeHeuristic{}) diff --git a/apps/backend-go/internal/providers/scrape/scrape.go b/apps/backend-go/internal/providers/scrape/scrape.go index 36bf8f304..1f8a01d98 100644 --- a/apps/backend-go/internal/providers/scrape/scrape.go +++ b/apps/backend-go/internal/providers/scrape/scrape.go @@ -59,10 +59,9 @@ type Mail interface { } // Browser is the part of the browser-service client the providers call -// directly (holehe, staffspy, the person_intel_deep page render). +// directly (staffspy, the person_intel_deep page render). type Browser interface { Render(ctx context.Context, req browserclient.RenderRequest) (browserclient.RenderResult, error) - Holehe(ctx context.Context, req browserclient.HoleheRequest) (browserclient.HoleheResult, error) StaffSpy(ctx context.Context, req browserclient.StaffSpyRequest) (browserclient.StaffSpyResult, error) } diff --git a/apps/backend-go/internal/providers/scrape/testdata/vectors.json b/apps/backend-go/internal/providers/scrape/testdata/vectors.json index b08e2487a..1fbdb12a6 100644 --- a/apps/backend-go/internal/providers/scrape/testdata/vectors.json +++ b/apps/backend-go/internal/providers/scrape/testdata/vectors.json @@ -2433,121 +2433,6 @@ "license": "unknown" } }, - { - "name": "hh_no_email", - "provider": "holehe", - "input": { - "email": "nope" - }, - "calls": [], - "output": { - "provider": "holehe", - "success": false, - "fields": {}, - "confidence": 0.0, - "error": "No email to verify", - "billing_evidence": null, - "license": "unknown" - } - }, - { - "name": "hh_real", - "provider": "holehe", - "input": { - "email": "a@b.com" - }, - "holehe": { - "exists_on": [ - "twitter", - "spotify", - "adobe", - "imgur", - "x", - "y" - ], - "total_checked": 50 - }, - "calls": [ - [ - "holehe", - "a@b.com" - ] - ], - "output": { - "provider": "holehe", - "success": true, - "fields": { - "email": "a@b.com", - "email_verify": "verified_active", - "email_presence": "twitter, spotify, adobe, imgur, x" - }, - "confidence": 0.95, - "error": "", - "billing_evidence": null, - "license": "unknown" - } - }, - { - "name": "hh_one", - "provider": "holehe", - "input": { - "email": "a@b.com" - }, - "holehe": { - "exists_on": [ - "github" - ], - "total_checked": 48 - }, - "calls": [ - [ - "holehe", - "a@b.com" - ] - ], - "output": { - "provider": "holehe", - "success": true, - "fields": { - "email": "a@b.com", - "email_verify": "likely_valid", - "email_presence": "github" - }, - "confidence": 0.65, - "error": "", - "billing_evidence": null, - "license": "unknown" - } - }, - { - "name": "hh_zero", - "provider": "holehe", - "input": { - "email": "a@b.com" - }, - "holehe": { - "exists_on": [], - "total_checked": 50 - }, - "calls": [ - [ - "holehe", - "a@b.com" - ] - ], - "output": { - "provider": "holehe", - "success": true, - "fields": { - "email": "a@b.com", - "email_verify": "no_presence" - }, - "confidence": 0.3, - "error": "", - "billing_evidence": null, - "license": "unknown" - } - }, { "name": "ss_no_company", "provider": "staffspy", diff --git a/apps/backend-go/internal/providers/scrape/unit_test.go b/apps/backend-go/internal/providers/scrape/unit_test.go index f0f677199..e5ae9f016 100644 --- a/apps/backend-go/internal/providers/scrape/unit_test.go +++ b/apps/backend-go/internal/providers/scrape/unit_test.go @@ -23,9 +23,6 @@ func (f *renderFake) Render(_ context.Context, r browserclient.RenderRequest) (b f.reqs = append(f.reqs, r) return f.fn(r) } -func (f *renderFake) Holehe(context.Context, browserclient.HoleheRequest) (browserclient.HoleheResult, error) { - return browserclient.HoleheResult{}, &browserclient.Error{Code: "busy"} -} func (f *renderFake) StaffSpy(context.Context, browserclient.StaffSpyRequest) (browserclient.StaffSpyResult, error) { return browserclient.StaffSpyResult{}, nil } @@ -100,7 +97,7 @@ func TestPersonIntelDeepFailures(t *testing.T) { func TestScrapeProvidersAreGated(t *testing.T) { for _, n := range []string{"website_scraper", "deep_scraper", "company_intel", "local_business", "crosslinked", "email_harvester", "facebook_pages", "social_finder", "ddg_company", "ddg_email", "decision_maker", "jobspy", - "holehe", "staffspy", "mailscout", "company_size_heuristic", "lead_scorer"} { + "staffspy", "mailscout", "company_size_heuristic", "lead_scorer"} { p, ok := providers.Get(n) if !ok { t.Fatalf("%s not registered", n) @@ -213,10 +210,6 @@ func TestStaffSpyOffWithoutOptIn(t *testing.T) { if out, _ := (staffSpy{}).Enrich(context.Background(), nil, nil, in); out.Error != staffspyUnavailable { t.Errorf("no li_at: %q", out.Error) } - Configure(Deps{Browser: &renderFake{}}) - if out, _ := (holehe{}).Enrich(context.Background(), nil, nil, providers.Input{Lead: map[string]any{"email": "a@b.co"}}); out.Error != "holehe unavailable: busy" { - t.Errorf("holehe: %q", out.Error) - } } func TestNoSeamIsAnErrorOutput(t *testing.T) { diff --git a/apps/backend-go/internal/providers/scrape/vectors_test.go b/apps/backend-go/internal/providers/scrape/vectors_test.go index df34f734d..a0f81435e 100644 --- a/apps/backend-go/internal/providers/scrape/vectors_test.go +++ b/apps/backend-go/internal/providers/scrape/vectors_test.go @@ -48,10 +48,6 @@ type scenario struct { Pick mailAns `json:"pick"` Verify mailAns `json:"verify"` } `json:"mail"` - Holehe struct { - ExistsOn []string `json:"exists_on"` - TotalChecked int `json:"total_checked"` - } `json:"holehe"` StaffSpy struct { Rows []browserclient.StaffRow `json:"rows"` } `json:"staffspy"` @@ -173,11 +169,6 @@ func (w *fakeWorld) Render(context.Context, browserclient.RenderRequest) (browse return browserclient.RenderResult{}, errors.New("render not scripted") } -func (w *fakeWorld) Holehe(_ context.Context, r browserclient.HoleheRequest) (browserclient.HoleheResult, error) { - w.log("holehe", r.Email) - return browserclient.HoleheResult{Email: r.Email, ExistsOn: w.sc.Holehe.ExistsOn, TotalChecked: w.sc.Holehe.TotalChecked}, nil -} - func (w *fakeWorld) StaffSpy(_ context.Context, r browserclient.StaffSpyRequest) (browserclient.StaffSpyResult, error) { w.log("staffspy", r.CompanyName, r.Location, r.MaxResults) return browserclient.StaffSpyResult{CompanyName: r.CompanyName, Staff: w.sc.StaffSpy.Rows}, nil diff --git a/apps/backend-go/internal/providers/testdata/python_provider_inventory.json b/apps/backend-go/internal/providers/testdata/python_provider_inventory.json index 2795b69cd..8041aec2a 100644 --- a/apps/backend-go/internal/providers/testdata/python_provider_inventory.json +++ b/apps/backend-go/internal/providers/testdata/python_provider_inventory.json @@ -483,36 +483,6 @@ "sha256": "3867d40be1587fe152bdbd718b46940b480849fc30fd3df10ed3aca589dd5207", "registered_in_workbook_registry": true }, - { - "module": "holehe_verify", - "classes": [ - { - "class": "HoleheProvider", - "bases": [ - "EnrichmentProvider" - ], - "name": "holehe", - "capabilities": [ - "email_verify" - ], - "cost_per_lookup": null, - "default_confidence": 0.85 - } - ], - "imports_top_level": [], - "imports_nested": [ - "holehe", - "httpx" - ], - "non_http_third_party": [ - "holehe" - ], - "uses_httpx": true, - "uses_smtp_or_dns": false, - "line_count": 144, - "sha256": "b93b5d864b323c56316b235f24864b766f00a0fa06aeb0e6f26085e0612a3583", - "registered_in_workbook_registry": true - }, { "module": "hunter_io", "classes": [ diff --git a/apps/backend-go/internal/vendorcatalog/native_costs.go b/apps/backend-go/internal/vendorcatalog/native_costs.go index b13b01516..24f81eea0 100644 --- a/apps/backend-go/internal/vendorcatalog/native_costs.go +++ b/apps/backend-go/internal/vendorcatalog/native_costs.go @@ -15,7 +15,6 @@ var nativeFreeProviders = map[string]bool{ "email_harvester": true, "facebook_pages": true, "gleif": true, - "holehe": true, "ipinfo": true, "jobspy": true, "jsonld_firmographics": true, diff --git a/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json b/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json index 7106a97e0..ae0bfb86c 100644 --- a/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json +++ b/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json @@ -156,13 +156,6 @@ ], "default_confidence": 0.9 }, - "holehe": { - "cost_per_lookup": 0.0, - "capabilities": [ - "email_verify" - ], - "default_confidence": 0.85 - }, "hunter_io": { "cost_per_lookup": 0.0, "capabilities": [ @@ -807,22 +800,6 @@ "has_known_cost": true, "is_byok": true }, - { - "name": "holehe", - "research_flag": false, - "base_cost": 0.0, - "is_paid": false, - "has_known_cost": true, - "is_byok": true - }, - { - "name": "holehe", - "research_flag": true, - "base_cost": 0.0, - "is_paid": false, - "has_known_cost": true, - "is_byok": true - }, { "name": "hunter_io", "research_flag": false, diff --git a/apps/backend-go/internal/wbregistry/registry.json b/apps/backend-go/internal/wbregistry/registry.json index a8178ea10..2d81d9e66 100644 --- a/apps/backend-go/internal/wbregistry/registry.json +++ b/apps/backend-go/internal/wbregistry/registry.json @@ -627,23 +627,6 @@ "maturity": "beta", "certification": null }, - { - "name": "holehe", - "capabilities": [ - "email_verify" - ], - "confidence": 0.85, - "required_checks": [ - "external_read", - "failure_recovery", - "normalization", - "provenance", - "quality_sample", - "tenant_isolation" - ], - "maturity": "beta", - "certification": null - }, { "name": "email_harvester", "capabilities": [ @@ -1443,8 +1426,7 @@ "type": "waterfall", "target_field": "email_verify", "waterfall": [ - "mailscout", - "holehe" + "mailscout" ], "condition": "{email} != \"\"", "width": 160 diff --git a/apps/backend-go/internal/workbook/cell.go b/apps/backend-go/internal/workbook/cell.go index 447a44182..3cda3c239 100644 --- a/apps/backend-go/internal/workbook/cell.go +++ b/apps/backend-go/internal/workbook/cell.go @@ -579,6 +579,15 @@ loop: skipped = append(skipped, skippedProvider{name, "unknown_provider"}) continue } + // A stored chain can name a provider that was removed from this build + // (e.g. holehe): skip it, like any unknown provider, instead of failing + // the run or stopping the waterfall before the later providers. A priced + // provider with no runner keeps its own path (released spend, cell fails). + if _, declarative := rs.p.Manifests[name]; !declarative && !rs.pl.paid(name) && !rs.registered(name) { + rs.log.Warn("provider not found, skipping", "provider", name) + skipped = append(skipped, skippedProvider{name, "unknown_provider"}) + continue + } reserved := rs.pl.paid(name) if !reserved && !rs.pl.knownCost(name) { resultError = "provider_price_unknown" diff --git a/apps/backend-go/internal/workbook/defaults.go b/apps/backend-go/internal/workbook/defaults.go index 6b01ad4eb..d36aa9a48 100644 --- a/apps/backend-go/internal/workbook/defaults.go +++ b/apps/backend-go/internal/workbook/defaults.go @@ -13,8 +13,8 @@ var defaultWaterfalls = map[string][]string{ "ddg_email", "mailscout", "hunter_io", "apollo_io", "snovio", "prospeo", }, - "email_confidence": {"mailscout", "holehe", "abstract_api", "debounce"}, - "email_verify": {"mailscout", "holehe", "abstract_api", "debounce"}, + "email_confidence": {"mailscout", "abstract_api", "debounce"}, + "email_verify": {"mailscout", "abstract_api", "debounce"}, "phone": { "deep_scraper", "jsonld_firmographics", "website_scraper", "local_business", "ddg_company", "facebook_pages", diff --git a/apps/backend-go/internal/workbook/provenance.go b/apps/backend-go/internal/workbook/provenance.go index 22fa902ab..9d0d246aa 100644 --- a/apps/backend-go/internal/workbook/provenance.go +++ b/apps/backend-go/internal/workbook/provenance.go @@ -28,7 +28,7 @@ var providerLicense = map[string]string{ "hunter_io": "proprietary-api", "apollo_io": "proprietary-api", "snovio": "proprietary-api", "prospeo": "proprietary-api", "people_data_labs": "proprietary-api", "clearbit": "proprietary-api", "abstract_api": "proprietary-api", "debounce": "proprietary-api", "deep_scraper": "scraped", "website_scraper": "scraped", "jsonld_firmographics": "scraped", "email_harvester": "scraped", - "ddg_email": "scraped", "ddg_company": "scraped", "mailscout": "scraped", "holehe": "scraped", "social_finder": "scraped", + "ddg_email": "scraped", "ddg_company": "scraped", "mailscout": "scraped", "social_finder": "scraped", "crosslinked": "scraped", "staffspy": "scraped", "decision_maker": "scraped", "local_business": "scraped", "facebook_pages": "scraped", "google_maps": "scraped", "tech_stack": "scraped", "company_intel": "scraped", "jobspy": "scraped", "ats_hiring": "scraped", "company_size_heuristic": "scraped", "lead_scorer": "scraped", diff --git a/apps/backend-go/internal/workbook/removed_provider_integration_test.go b/apps/backend-go/internal/workbook/removed_provider_integration_test.go new file mode 100644 index 000000000..27f1a5339 --- /dev/null +++ b/apps/backend-go/internal/workbook/removed_provider_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package workbook_test + +import ( + "context" + "testing" + "time" + + "github.com/debpalash/OpenGTM/apps/backend-go/internal/provcall" + "github.com/debpalash/OpenGTM/apps/backend-go/internal/providers" + "github.com/debpalash/OpenGTM/apps/backend-go/internal/workbook" +) + +// removedRunner serves only "ok"; every other name is neither a Go provider nor +// a manifest, like a provider that was removed from the build (holehe). +type removedRunner struct{ calls []string } + +func (r *removedRunner) Serves(name string) bool { return name == "ok" } + +func (r *removedRunner) Run(_ context.Context, _, name string, _ providers.Input, _ time.Duration) (providers.Output, error) { + r.calls = append(r.calls, name) + if name != "ok" { + return providers.Output{}, provcall.ErrUnavailable + } + f := providers.NewObj() + f.Set(target, "7") + return providers.Output{Extra: f, Confidence: 0.9}, nil +} + +// A stored waterfall that still names a removed provider must skip it and let +// the later providers run, not fail the run or stop the waterfall. +func TestStoredChainNamingRemovedProviderIsSkipped(t *testing.T) { + e := setup(t, wsA) + e.seed(wsA, "wb-gone", "["+colJSON("c1", "holehe", "ok")+"]", 1, 0) + raw := mutate(payloadFor(wsA, "wb-gone", map[string]float64{"holehe": 0, "ok": 0}, nil), func(p map[string]any) { + delete(p["go"].(map[string]any)["manifests"].(map[string]any), "holehe") + }) + r := &removedRunner{} + d := e.deps(newSim(t, nil), nil) + d.Runner = r + res, err := workbook.Run(e.ctx, d, e.claim(wsA, raw)) + if err != nil || res.Completed != 1 || res.Errors != 0 { + t.Fatalf("%+v %v", res, err) + } + if len(r.calls) != 1 || r.calls[0] != "ok" { + t.Fatalf("the removed provider must never be called: %v", r.calls) + } + if v := e.str(wsA, `SELECT value FROM workbook_enrichments WHERE column_id='c1'`); v != "7" { + t.Fatalf("value %q", v) + } +} diff --git a/apps/backend-go/internal/workbook/runner.go b/apps/backend-go/internal/workbook/runner.go index 48c9320c5..c89e5d434 100644 --- a/apps/backend-go/internal/workbook/runner.go +++ b/apps/backend-go/internal/workbook/runner.go @@ -105,6 +105,16 @@ func (rs *runState) serves(name string) bool { return true } +// registered reports whether name is a provider this build knows: a Go-native +// provider, or one the runner serves (a verified manifest). A name that is +// neither was removed or never existed. +func (rs *runState) registered(name string) bool { + if _, ok := providers.Get(name); ok { + return true + } + return rs.serves(name) +} + // precheck refuses a call the runner can already tell will not reach the network. func (rs *runState) precheck(ctx context.Context, name string, lead map[string]any) string { if pc, ok := rs.runner.(prechecker); ok { diff --git a/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json b/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json index 9e0be9c1b..790aa4243 100644 --- a/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json +++ b/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json @@ -5819,7 +5819,6 @@ "ddg_email": "scraped", "ddg_company": "scraped", "mailscout": "scraped", - "holehe": "scraped", "social_finder": "scraped", "crosslinked": "scraped", "staffspy": "scraped", diff --git a/apps/backend-go/testdata/contract/meta/providers.json b/apps/backend-go/testdata/contract/meta/providers.json index aa861a7e2..e3c7c4d97 100644 --- a/apps/backend-go/testdata/contract/meta/providers.json +++ b/apps/backend-go/testdata/contract/meta/providers.json @@ -458,23 +458,6 @@ "tenant_isolation" ] }, - { - "capabilities": [ - "email_verify" - ], - "certification": null, - "confidence": 0.85, - "maturity": "beta", - "name": "holehe", - "required_checks": [ - "external_read", - "failure_recovery", - "normalization", - "provenance", - "quality_sample", - "tenant_isolation" - ] - }, { "capabilities": [ "email", diff --git a/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json b/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json index 3c2978ffc..dec8621a8 100644 --- a/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json +++ b/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json @@ -457,23 +457,6 @@ "tenant_isolation" ] }, - { - "capabilities": [ - "email_verify" - ], - "certification": null, - "confidence": 0.85, - "maturity": "beta", - "name": "holehe", - "required_checks": [ - "external_read", - "failure_recovery", - "normalization", - "provenance", - "quality_sample", - "tenant_isolation" - ] - }, { "capabilities": [ "email", diff --git a/apps/browser-service/pyproject.toml b/apps/browser-service/pyproject.toml index d82389528..8f44c05f1 100644 --- a/apps/browser-service/pyproject.toml +++ b/apps/browser-service/pyproject.toml @@ -8,7 +8,6 @@ dependencies = [ "curl-cffi>=0.17.0", "ddgs>=9.16.1", "fastapi>=0.162.0", - "holehe>=1.61", "httpx>=0.28.1", "lxml>=6.1.3", "markdownify>=1.2.3", diff --git a/apps/browser-service/scripts/record_contract_fixtures.py b/apps/browser-service/scripts/record_contract_fixtures.py index fd67c3aa6..4a0a624a2 100644 --- a/apps/browser-service/scripts/record_contract_fixtures.py +++ b/apps/browser-service/scripts/record_contract_fixtures.py @@ -2,7 +2,7 @@ uv run python scripts/record_contract_fixtures.py ../backend-go/internal/browserclient/testdata/recorded -Runs the FastAPI app in-process with fake browser / ddgs / holehe / staffspy +Runs the FastAPI app in-process with fake browser / ddgs / staffspy backends (no network) and saves request + response pairs; extract runs the real trafilatura and markdownify. """ @@ -19,7 +19,7 @@ from browser_service.app import create_app from browser_service.config import Settings -from browser_service.scrapers import holehe_check, staffspy_roster +from browser_service.scrapers import staffspy_roster TOKEN = "r" * 40 ARTICLE = ( @@ -79,19 +79,6 @@ async def fake_search(deps, method, search, max_results, deadline, **opts): async def main(out: Path) -> None: - async def fn(email, client, o, _n="twitter"): - o.append({"name": _n, "domain": "twitter.com", "method": "register", "rateLimit": False, "exists": True}) - - async def fn2(email, client, o): - o.append({"name": "spotify", "domain": "spotify.com", "method": "register", "rateLimit": True, "exists": False}) - - holehe_check.load_modules = lambda: [fn, fn2] - - class C: - async def aclose(self): - pass - - holehe_check.make_client = lambda p, t: C() staffspy_roster.check_logged_in = lambda s: True staffspy_roster.scrape_staff = lambda s, c, t, loc, n: [ { @@ -109,7 +96,6 @@ async def aclose(self): "search_error": ("POST", "/v1/search", {"query": "fail", "max_results": 2}), "extract": ("POST", "/v1/extract", {"html": ARTICLE, "url": "https://acme.example/a"}), "extract_url": ("POST", "/v1/extract", {"url": "https://acme.example/a", "engine": "browser"}), - "holehe": ("POST", "/v1/holehe", {"email": "jane@acme.example"}), "staffspy": ( "POST", "/v1/staffspy", diff --git a/apps/browser-service/src/browser_service/app.py b/apps/browser-service/src/browser_service/app.py index d7393713b..5fd76799b 100644 --- a/apps/browser-service/src/browser_service/app.py +++ b/apps/browser-service/src/browser_service/app.py @@ -40,8 +40,6 @@ GoogleMapsResult, GoogleSearchRequest, Health, - HoleheRequest, - HoleheResult, JobBoardsRequest, JustDialPageRequest, JustDialPageResult, @@ -61,7 +59,7 @@ Stats, WebDirectoriesRequest, ) -from .scrapers import google_maps, holehe_check, justdial, search, staffspy_roster +from .scrapers import google_maps, justdial, search, staffspy_roster logger = logging.getLogger("browser_service.app") @@ -329,10 +327,6 @@ async def search_query(req: SearchRequest) -> SearchOut: async def extract_content(req: ExtractRequest) -> ExtractResult: return await run("extract", req.timeout_s, lambda d: extract.extract(deps, req, d)) - @v1.post("/holehe", response_model=HoleheResult) - async def holehe(req: HoleheRequest) -> HoleheResult: - return await run("holehe", req.timeout_s, lambda d: holehe_check.check(deps, req, d)) - @v1.post("/staffspy", response_model=StaffSpyResult) async def staffspy(req: StaffSpyRequest) -> StaffSpyResult: return await run("staffspy", req.timeout_s, lambda d: staffspy_roster.scrape(deps, req, d)) diff --git a/apps/browser-service/src/browser_service/models.py b/apps/browser-service/src/browser_service/models.py index 2cd44fb3b..da0e109ab 100644 --- a/apps/browser-service/src/browser_service/models.py +++ b/apps/browser-service/src/browser_service/models.py @@ -325,33 +325,6 @@ class ExtractResult(BaseModel): elapsed_ms: int -# ── holehe ─────────────────────────────────────────────────────────── - - -class HoleheRequest(_Upstream): - email: str = Field(min_length=3, max_length=254, pattern=r"^[^@\s]+@[^@\s]+\.[^@\s]+$") - max_modules: int = Field(default=50, ge=1, le=200, description="legacy checked the first 50 modules") - timeout_s: float = Field(default=60.0, ge=5.0, le=300.0) - - -class HoleheSite(BaseModel): - name: str - domain: str - method: str - exists: bool - rate_limited: bool - - -class HoleheResult(BaseModel): - email: str - sites: list[HoleheSite] - exists_on: list[str] - total_checked: int - errors: int = Field(description="modules that raised") - partial: bool - elapsed_ms: int - - # ── staffspy ───────────────────────────────────────────────────────── diff --git a/apps/browser-service/src/browser_service/scrapers/holehe_check.py b/apps/browser-service/src/browser_service/scrapers/holehe_check.py deleted file mode 100644 index 15dc89a23..000000000 --- a/apps/browser-service/src/browser_service/scrapers/holehe_check.py +++ /dev/null @@ -1,116 +0,0 @@ -"""POST /v1/holehe: which sites an email is registered on (legacy -leadgen/enrichment/providers/holehe_verify.py). - -Same module set and order as legacy (pkgutil walk of holehe.modules, first 50 -by default) and the same concurrency (10) and per-request timeout (10 s). Every -module's traffic goes through the egress proxy. The confidence / verdict -mapping (is_real, email_verify) stays in Go; this returns raw per-site results. - -Deviation (legacy bug, fixed): legacy passed a dict as holehe's ``out`` list, -so every module raised on ``out.append`` and no site was ever reported. -""" - -from __future__ import annotations - -import asyncio -import importlib -import logging -import pkgutil -from functools import cache -from typing import Any - -from ..engines import Deadline, Deps -from ..models import HoleheRequest, HoleheResult, HoleheSite - -logger = logging.getLogger("browser_service.scrapers.holehe") - -CONCURRENCY = 10 -CLIENT_TIMEOUT_S = 10 - - -@cache -def _modules() -> tuple: - import holehe.modules - - out = [] - for _imp, modname, ispkg in pkgutil.walk_packages(holehe.modules.__path__, holehe.modules.__name__ + "."): - if ispkg: - continue - try: - mod = importlib.import_module(modname) - except Exception: # noqa: S112 - a broken module is skipped, as in legacy - continue - fn = getattr(mod, modname.rsplit(".", 1)[-1], None) - if fn is not None: - out.append(fn) - return tuple(out) - - -def load_modules() -> list: - return list(_modules()) - - -def make_client(proxy_url: str, timeout: float) -> Any: - import httpx - - return httpx.AsyncClient(proxy=proxy_url, timeout=timeout, trust_env=False, follow_redirects=False) - - -async def check(deps: Deps, req: HoleheRequest, deadline: Deadline) -> HoleheResult: - modules = load_modules()[: req.max_modules] - session = deps.proxy.open_session(deps.settings.max_download_bytes, upstream=req.upstream()) - client = make_client(deps.proxy.proxy_url(session), CLIENT_TIMEOUT_S) - sem = asyncio.Semaphore(CONCURRENCY) - errors = 0 - - async def one(fn) -> dict | None: - nonlocal errors - async with sem: - out: list = [] - try: - await fn(req.email, client, out) - except Exception: - errors += 1 - return None - return out[0] if out and isinstance(out[0], dict) else None - - tasks = [asyncio.ensure_future(one(fn)) for fn in modules] - partial = False - try: - done, pending = await asyncio.wait(tasks, timeout=max(0.5, deadline.left() - 1.0)) if tasks else (set(), set()) - if pending: - partial = True - for t in pending: - t.cancel() - await asyncio.gather(*pending, return_exceptions=True) - finally: - try: - await client.aclose() - finally: - deps.proxy.close_session(session) - - sites: list[HoleheSite] = [] - for t in tasks: # module order, as legacy gathered them - if t.cancelled() or t.exception() is not None: - continue - r = t.result() - if r is None: - continue - sites.append( - HoleheSite( - name=str(r.get("name", "unknown"))[:100], - domain=str(r.get("domain") or "")[:253], - method=str(r.get("method") or "")[:50], - exists=r.get("exists") is True, - rate_limited=bool(r.get("rateLimit")), - ) - ) - return HoleheResult( - email=req.email, - sites=sites, - exists_on=[s.name for s in sites if s.exists], - total_checked=len(sites), - errors=errors, - partial=partial, - elapsed_ms=deadline.elapsed_ms(), - ) diff --git a/apps/browser-service/tests/test_ops.py b/apps/browser-service/tests/test_ops.py index 1acac55c4..adf311157 100644 --- a/apps/browser-service/tests/test_ops.py +++ b/apps/browser-service/tests/test_ops.py @@ -1,8 +1,7 @@ """Ops added for the Go web layer (plan 15 WP-WEB): generic search, extract, -holehe, staffspy, stats, per-request upstream proxies and the stealth Maps +staffspy, stats, per-request upstream proxies and the stealth Maps browser.""" -import asyncio import sys import types @@ -11,7 +10,7 @@ from browser_service.config import ConfigError, Settings from browser_service.egress import UpstreamProxy -from browser_service.scrapers import holehe_check, search, staffspy_roster +from browser_service.scrapers import search, staffspy_roster ARTICLE = ( "Acme Robotics raises Series B" @@ -235,89 +234,6 @@ async def test_extract_accepts_bodies_above_the_generic_cap(make_client): assert r.status_code == 413 -# ── /v1/holehe ─────────────────────────────────────────────────────── - - -def _fake_holehe(monkeypatch, behaviours): - """Install fake holehe modules: name -> exists | 'rate' | Exception.""" - funcs = [] - for name, b in behaviours.items(): - - async def fn(email, client, out, _name=name, _b=b): - if isinstance(_b, Exception): - raise _b - out.append( - { - "name": _name, - "domain": f"{_name}.com", - "method": "register", - "frequent_rate_limit": False, - "rateLimit": _b == "rate", - "exists": _b is True, - "emailrecovery": "secret-recovery", - "phoneNumber": None, - "others": None, - } - ) - - fn.__name__ = name - funcs.append(fn) - monkeypatch.setattr(holehe_check, "load_modules", lambda: funcs) - return funcs - - -async def test_holehe_reports_per_site_results(make_client, monkeypatch): - _fake_holehe(monkeypatch, {"twitter": True, "spotify": False, "instagram": "rate", "adobe": RuntimeError("x")}) - seen = {} - - def fake_client(proxy_url, timeout): - seen["proxy"] = proxy_url - seen["timeout"] = timeout - - class C: - async def aclose(self): - seen["closed"] = True - - return C() - - monkeypatch.setattr(holehe_check, "make_client", fake_client) - c = await make_client() - r = await c.post("/v1/holehe", headers=AUTH, json={"email": "jane@acme.example"}) - assert r.status_code == 200, r.text - b = r.json() - assert b["exists_on"] == ["twitter"] and b["total_checked"] == 3 and b["errors"] == 1 - assert [s["name"] for s in b["sites"]] == ["twitter", "spotify", "instagram"] - assert b["sites"][2]["rate_limited"] is True and b["sites"][0]["domain"] == "twitter.com" - assert "secret-recovery" not in r.text - assert seen["proxy"].startswith("http://s") and seen["closed"] and seen["timeout"] == 10 - - -async def test_holehe_caps_modules(make_client, monkeypatch): - _fake_holehe(monkeypatch, {f"m{i}": False for i in range(60)}) - monkeypatch.setattr(holehe_check, "make_client", lambda p, t: types.SimpleNamespace(aclose=_anoop)) - c = await make_client() - r = await c.post("/v1/holehe", headers=AUTH, json={"email": "a@b.example"}) - assert r.json()["total_checked"] == 50 - r = await c.post("/v1/holehe", headers=AUTH, json={"email": "a@b.example", "max_modules": 5}) - assert r.json()["total_checked"] == 5 - - -async def _anoop(): - return None - - -@pytest.mark.parametrize("email", ["", "no-at-sign", "a@b", "x" * 300 + "@a.com"]) -async def test_holehe_validation(make_client, email): - c = await make_client() - r = await c.post("/v1/holehe", headers=AUTH, json={"email": email}) - assert r.status_code == 400 - - -def test_holehe_real_modules_load(): - mods = holehe_check.load_modules() - assert len(mods) > 100 and all(asyncio.iscoroutinefunction(m) for m in mods[:5]) - - # ── /v1/staffspy ───────────────────────────────────────────────────── LI = [ diff --git a/apps/browser-service/uv.lock b/apps/browser-service/uv.lock index 2edfe04eb..2ef378f85 100644 --- a/apps/browser-service/uv.lock +++ b/apps/browser-service/uv.lock @@ -45,15 +45,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, ] -[[package]] -name = "attrs" -version = "26.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, -] - [[package]] name = "babel" version = "2.18.0" @@ -85,7 +76,6 @@ dependencies = [ { name = "curl-cffi" }, { name = "ddgs" }, { name = "fastapi" }, - { name = "holehe" }, { name = "httpx" }, { name = "lxml" }, { name = "markdownify" }, @@ -109,7 +99,6 @@ requires-dist = [ { name = "curl-cffi", specifier = ">=0.17.0" }, { name = "ddgs", specifier = ">=9.16.1" }, { name = "fastapi", specifier = ">=0.162.0" }, - { name = "holehe", specifier = ">=1.61" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "lxml", specifier = ">=6.1.3" }, { name = "markdownify", specifier = ">=1.2.3" }, @@ -127,18 +116,6 @@ dev = [ { name = "ruff", specifier = ">=0.17.3" }, ] -[[package]] -name = "bs4" -version = "0.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c9/aa/4acaf814ff901145da37332e05bb510452ebed97bc9602695059dd46ef39/bs4-0.0.2.tar.gz", hash = "sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925", size = 698, upload-time = "2024-01-17T18:15:47.371Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/bb/bf7aab772a159614954d84aa832c129624ba6c32faa559dfb200a534e50b/bs4-0.0.2-py2.py3-none-any.whl", hash = "sha256:abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc", size = 1189, upload-time = "2024-01-17T18:15:48.613Z" }, -] - [[package]] name = "certifi" version = "2026.10.23" @@ -531,20 +508,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] -[[package]] -name = "holehe" -version = "1.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "bs4" }, - { name = "colorama" }, - { name = "httpx" }, - { name = "termcolor" }, - { name = "tqdm" }, - { name = "trio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/86/2793b1d6139a54903d59d68aec364fe92260ddaa2e27c938dcace62a1eb0/holehe-1.61.tar.gz", hash = "sha256:1a02e7c103b9abcb485eb0aaca36d950ac6bff4ec001d644362cc46739916b3b", size = 44563, upload-time = "2022-07-21T14:22:36.62Z" } - [[package]] name = "htmldate" version = "1.11.0" @@ -888,18 +851,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, ] -[[package]] -name = "outcome" -version = "1.3.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" }, -] - [[package]] name = "packaging" version = "26.3" @@ -1409,24 +1360,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "sniffio" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, -] - -[[package]] -name = "sortedcontainers" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, -] - [[package]] name = "soupsieve" version = "3.0.4" @@ -1476,15 +1409,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/3f/8ba87d9e287b9d385a02a7114ddcef61b26f86411e121c9003eb509a1773/tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687", size = 28165, upload-time = "2024-07-05T07:25:29.591Z" }, ] -[[package]] -name = "termcolor" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, -] - [[package]] name = "tld" version = "0.13.2" @@ -1509,18 +1433,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/e0/d5760e222a7e3f3aec7ef59f6dff952af27d09ec168bcca50751d9698151/tldextract-5.4.0-py3-none-any.whl", hash = "sha256:7f02aed30bd3b6ad5717192eb859a39b20aafc7caf3917d9cf6cb00a58efb34f", size = 107482, upload-time = "2026-10-03T21:32:45.842Z" }, ] -[[package]] -name = "tqdm" -version = "4.70.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, -] - [[package]] name = "trafilatura" version = "2.3.1" @@ -1539,23 +1451,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/20/e95d135c417aceaa3d34d1b4dff9aceb85e98800480eca1b033933717169/trafilatura-2.3.1-py3-none-any.whl", hash = "sha256:f86bad2ee36f82884e14dee84c83aae0f0ac0127da5c65764e82f3846716b822", size = 158660, upload-time = "2026-10-06T19:30:24.385Z" }, ] -[[package]] -name = "trio" -version = "0.34.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" }, - { name = "idna" }, - { name = "outcome" }, - { name = "sniffio" }, - { name = "sortedcontainers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/92/dc/a2d25ed73ad49cfd79bf18d262577c3731c98e382284e28d522f49a0df35/trio-0.34.0.tar.gz", hash = "sha256:63b9485408bdfdde544fced107045a8c0086cdc4bd0ef2f797b9e0dd111b964b", size = 607457, upload-time = "2026-08-11T00:33:42.198Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/1f/555f1364bed52a92a864181962b77f1b15adadeacf23b86105324363e461/trio-0.34.0-py3-none-any.whl", hash = "sha256:6c7c9f49917694dcdcd5f67abd168df5599eca480d61f29854d17a61a75c2f05", size = 511840, upload-time = "2026-08-11T00:33:40.552Z" }, -] - [[package]] name = "typing-extensions" version = "4.16.0" diff --git a/apps/docs/src/content/docs/concepts/waterfalls.md b/apps/docs/src/content/docs/concepts/waterfalls.md index b5efca87e..bb24d5a62 100644 --- a/apps/docs/src/content/docs/concepts/waterfalls.md +++ b/apps/docs/src/content/docs/concepts/waterfalls.md @@ -19,7 +19,7 @@ deep_scraper → jsonld_firmographics → website_scraper → email_harvester → ddg_email → mailscout → hunter_io → apollo_io → snovio → prospeo ``` -For `email_verify`: `mailscout → holehe → abstract_api → debounce`. For +For `email_verify`: `mailscout → abstract_api → debounce`. For `mobile_phone`, which has no free tier: `leadmagic_mobile → prospeo_mobile`, cost-ordered. Chains exist for phone, description, decision makers, contact person and title, social URLs, company size, industry tags, address, founding diff --git a/apps/docs/src/content/docs/guides/providers.md b/apps/docs/src/content/docs/guides/providers.md index bb965ee8f..6a3d6b9f4 100644 --- a/apps/docs/src/content/docs/guides/providers.md +++ b/apps/docs/src/content/docs/guides/providers.md @@ -38,7 +38,6 @@ The last four rows in the first group (`leadmagic_*`, `prospeo_mobile`) are | Provider id | Fills | |---|---| | `mailscout` | SMTP email verification (built in) | -| `holehe` | email existence across 120+ sites | | `ddg_email`, `ddg_company` | email, website, phone, description via search | | `email_harvester`, `website_scraper`, `deep_scraper` | emails, phones, socials, description scraped from the company site | | `jsonld_firmographics` | schema.org firmographics | diff --git a/docs/go-port/runbooks/browser-service.md b/docs/go-port/runbooks/browser-service.md index c840c1215..f26da1a5f 100644 --- a/docs/go-port/runbooks/browser-service.md +++ b/docs/go-port/runbooks/browser-service.md @@ -29,11 +29,10 @@ Go reaches it only through `internal/browserclient`. | `POST /v1/scrape/{linkedin,job_boards,crunchbase,review_directories,web_directories,google_search}` | Raw search hits per search string. The search strings are the exact legacy templates. | | `POST /v1/search` | One keyless ddgs query (text or news) with the caller's backends, region, safesearch and timelimit. Failures come back as `error`: `no_results`, `rate_limited`, `timeout` or `failed` (rate_limited and timeout are the legacy block signals). | | `POST /v1/extract` | Main text and metadata (trafilatura, legacy content_extract) plus markdown (markdownify) from given HTML or a fetched URL. Its body cap is `BROWSER_MAX_EXTRACT_BYTES`. | -| `POST /v1/holehe` | Per-site holehe results for one email (first 50 modules, concurrency 10, as legacy). The verdict mapping stays in Go. | | `POST /v1/staffspy` | A company's LinkedIn roster. Off (503 `engine_unavailable`) unless `BROWSER_STAFFSPY_ENABLED`. The LinkedIn cookies come with each request; nothing is written to disk. | | `GET /v1/stats` | This process's counters: calls, ok and error codes per op, search failures by class, egress sessions, refusals and upstream failures. Per replica, not aggregated. | -Every page, search, holehe and staffspy request takes an optional `upstream_proxy` (http, socks5 or socks4 URL). +Every page, search and staffspy request takes an optional `upstream_proxy` (http, socks5 or socks4 URL). Go picks it from `PROXY_LIST` or the proxy manager; the service never stores or logs it. See "Upstream proxies" below. Errors use the envelope `{"error":{"code","message","fields?"}}`. Each code has one fixed message, so page text