From 6a53fb397ef410001e6b10479275373dab5a93c3 Mon Sep 17 00:00:00 2001 From: Shimul Debnath <270455167+velixio@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:36:13 +0530 Subject: [PATCH] chore(providers): remove the GPL holehe provider holehe is GPL-3.0 and the browser service is to become closed source. Remove the /v1/holehe route, models, scraper, dependency (and the packages only it pulled in), the Go provider, browserclient method and types, defaults, provenance, catalog, registry and contract fixtures, and the live docs. A stored workbook chain that still names holehe is now skipped as unknown_provider instead of failing the run or stopping the waterfall. Signed-off-by: Shimul Debnath <270455167+velixio@users.noreply.github.com> --- .../internal/browserclient/client.go | 7 - .../internal/browserclient/contract_test.go | 2 - .../internal/browserclient/recorded_test.go | 16 +- .../browserclient/testdata/openapi.json | 247 ------------------ .../testdata/recorded/holehe.json | 33 --- .../testdata/recorded/stats.json | 5 - .../internal/browserclient/types.go | 39 --- .../internal/providers/scrape/library.go | 44 ---- .../internal/providers/scrape/register.go | 1 - .../internal/providers/scrape/scrape.go | 3 +- .../providers/scrape/testdata/vectors.json | 115 -------- .../internal/providers/scrape/unit_test.go | 9 +- .../internal/providers/scrape/vectors_test.go | 9 - .../testdata/python_provider_inventory.json | 30 --- .../internal/vendorcatalog/native_costs.go | 1 - .../testdata/catalog_vectors.json | 23 -- .../internal/wbregistry/registry.json | 20 +- apps/backend-go/internal/workbook/cell.go | 9 + apps/backend-go/internal/workbook/defaults.go | 4 +- .../internal/workbook/provenance.go | 2 +- .../removed_provider_integration_test.go | 52 ++++ apps/backend-go/internal/workbook/runner.go | 10 + .../workbook/testdata/wbexec_vectors.json | 1 - .../testdata/contract/meta/providers.json | 17 -- .../contract/meta/providers_no_ws_header.json | 17 -- apps/browser-service/pyproject.toml | 1 - .../scripts/record_contract_fixtures.py | 18 +- .../src/browser_service/app.py | 8 +- .../src/browser_service/models.py | 27 -- .../browser_service/scrapers/holehe_check.py | 116 -------- apps/browser-service/tests/test_ops.py | 88 +------ apps/browser-service/uv.lock | 105 -------- .../src/content/docs/concepts/waterfalls.md | 2 +- .../docs/src/content/docs/guides/providers.md | 1 - docs/go-port/runbooks/browser-service.md | 3 +- 35 files changed, 86 insertions(+), 999 deletions(-) delete mode 100644 apps/backend-go/internal/browserclient/testdata/recorded/holehe.json create mode 100644 apps/backend-go/internal/workbook/removed_provider_integration_test.go delete mode 100644 apps/browser-service/src/browser_service/scrapers/holehe_check.py diff --git a/apps/backend-go/internal/browserclient/client.go b/apps/backend-go/internal/browserclient/client.go index 4b85a87ed..2e439efca 100644 --- a/apps/backend-go/internal/browserclient/client.go +++ b/apps/backend-go/internal/browserclient/client.go @@ -296,13 +296,6 @@ func (c *Client) Extract(ctx context.Context, req ExtractRequest) (ExtractResult return out, c.post(ctx, "/v1/extract", req, req.Timeout, &out) } -// Holehe checks which sites an email is registered on. -func (c *Client) Holehe(ctx context.Context, req HoleheRequest) (HoleheResult, error) { - req.Timeout = clamp(req.Timeout, 60*time.Second) - var out HoleheResult - return out, c.post(ctx, "/v1/holehe", req, req.Timeout, &out) -} - // StaffSpy scrapes a company's LinkedIn roster. func (c *Client) StaffSpy(ctx context.Context, req StaffSpyRequest) (StaffSpyResult, error) { req.Timeout = clamp(req.Timeout, 180*time.Second) diff --git a/apps/backend-go/internal/browserclient/contract_test.go b/apps/backend-go/internal/browserclient/contract_test.go index cc77efecf..076b17fed 100644 --- a/apps/backend-go/internal/browserclient/contract_test.go +++ b/apps/backend-go/internal/browserclient/contract_test.go @@ -323,8 +323,6 @@ func endpoints() []endpoint { {"post", "/v1/extract", ExtractRequest{HTML: "
x
", URL: "https://e.com/a", Engine: "browser", IncludeMarkdown: true, MaxBytes: 4096, UpstreamProxy: "http://1.2.3.4:8080", Timeout: 20 * time.Second}, "ExtractRequest", &ExtractResult{}, "ExtractResult"}, - {"post", "/v1/holehe", HoleheRequest{Email: "a@b.example", MaxModules: 10, UpstreamProxy: "http://1.2.3.4:8080", - Timeout: 30 * time.Second}, "HoleheRequest", &HoleheResult{}, "HoleheResult"}, {"post", "/v1/staffspy", StaffSpyRequest{CompanyName: "Acme", SearchTerm: "ceo", Location: "Pune", MaxResults: 5, Cookies: li, UpstreamProxy: "http://1.2.3.4:8080", Timeout: 60 * time.Second}, "StaffSpyRequest", &StaffSpyResult{}, "StaffSpyResult"}, diff --git a/apps/backend-go/internal/browserclient/recorded_test.go b/apps/backend-go/internal/browserclient/recorded_test.go index bb5a174c2..e086be5a1 100644 --- a/apps/backend-go/internal/browserclient/recorded_test.go +++ b/apps/backend-go/internal/browserclient/recorded_test.go @@ -121,21 +121,9 @@ func TestRecordedExtract(t *testing.T) { } } -func TestRecordedHoleheStaffSpyStats(t *testing.T) { - f := loadFixture(t, "holehe") +func TestRecordedStaffSpyStats(t *testing.T) { + f := loadFixture(t, "staffspy") c, sent := replay(t, f) - h, err := c.Holehe(context.Background(), HoleheRequest{Email: "jane@acme.example"}) - if err != nil { - t.Fatal(err) - } - sameRequest(t, "holehe", *sent, f.Request) - strict(t, "holehe", f.Response, &HoleheResult{}) - if !reflect.DeepEqual(h.ExistsOn, []string{"twitter"}) || h.TotalChecked != 2 || !h.Sites[1].RateLimited { - t.Fatalf("%+v", h) - } - - f = loadFixture(t, "staffspy") - c, sent = replay(t, f) s, err := c.StaffSpy(context.Background(), StaffSpyRequest{CompanyName: "Acme", Cookies: []Cookie{{Name: "li_at", Value: "x", Domain: ".linkedin.com"}}}) if err != nil { diff --git a/apps/backend-go/internal/browserclient/testdata/openapi.json b/apps/backend-go/internal/browserclient/testdata/openapi.json index d49e3fe42..787b5a2ba 100644 --- a/apps/backend-go/internal/browserclient/testdata/openapi.json +++ b/apps/backend-go/internal/browserclient/testdata/openapi.json @@ -624,135 +624,6 @@ "title": "Health", "type": "object" }, - "HoleheRequest": { - "additionalProperties": false, - "properties": { - "email": { - "maxLength": 254, - "minLength": 3, - "pattern": "^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", - "title": "Email", - "type": "string" - }, - "max_modules": { - "default": 50, - "description": "legacy checked the first 50 modules", - "maximum": 200.0, - "minimum": 1.0, - "title": "Max Modules", - "type": "integer" - }, - "timeout_s": { - "default": 60.0, - "maximum": 300.0, - "minimum": 5.0, - "title": "Timeout S", - "type": "number" - }, - "upstream_proxy": { - "anyOf": [ - { - "maxLength": 2048, - "minLength": 1, - "type": "string" - }, - { - "type": "null" - } - ], - "description": "upstream proxy (http/socks5/socks4 URL) the caller chose for this request; egress still goes through the service's guard first. Never logged or stored.", - "title": "Upstream Proxy" - } - }, - "required": [ - "email" - ], - "title": "HoleheRequest", - "type": "object" - }, - "HoleheResult": { - "properties": { - "elapsed_ms": { - "title": "Elapsed Ms", - "type": "integer" - }, - "email": { - "title": "Email", - "type": "string" - }, - "errors": { - "description": "modules that raised", - "title": "Errors", - "type": "integer" - }, - "exists_on": { - "items": { - "type": "string" - }, - "title": "Exists On", - "type": "array" - }, - "partial": { - "title": "Partial", - "type": "boolean" - }, - "sites": { - "items": { - "$ref": "#/components/schemas/HoleheSite" - }, - "title": "Sites", - "type": "array" - }, - "total_checked": { - "title": "Total Checked", - "type": "integer" - } - }, - "required": [ - "email", - "sites", - "exists_on", - "total_checked", - "errors", - "partial", - "elapsed_ms" - ], - "title": "HoleheResult", - "type": "object" - }, - "HoleheSite": { - "properties": { - "domain": { - "title": "Domain", - "type": "string" - }, - "exists": { - "title": "Exists", - "type": "boolean" - }, - "method": { - "title": "Method", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "rate_limited": { - "title": "Rate Limited", - "type": "boolean" - } - }, - "required": [ - "name", - "domain", - "method", - "exists", - "rate_limited" - ], - "title": "HoleheSite", - "type": "object" - }, "JobBoardsRequest": { "additionalProperties": false, "properties": { @@ -2338,124 +2209,6 @@ "summary": "Health" } }, - "/v1/holehe": { - "post": { - "operationId": "holehe_v1_holehe_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HoleheRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HoleheResult" - } - } - }, - "description": "Successful Response" - }, - "400": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Bad Request" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Unauthorized" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Forbidden" - }, - "413": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Content Too Large" - }, - "429": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Too Many Requests" - }, - "500": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Internal Server Error" - }, - "502": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Bad Gateway" - }, - "503": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Service Unavailable" - }, - "504": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorEnvelope" - } - } - }, - "description": "Gateway Timeout" - } - }, - "summary": "Holehe" - } - }, "/v1/render": { "post": { "operationId": "render_v1_render_post", diff --git a/apps/backend-go/internal/browserclient/testdata/recorded/holehe.json b/apps/backend-go/internal/browserclient/testdata/recorded/holehe.json deleted file mode 100644 index 87e2dba33..000000000 --- a/apps/backend-go/internal/browserclient/testdata/recorded/holehe.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "method": "POST", - "path": "/v1/holehe", - "request": { - "email": "jane@acme.example" - }, - "response": { - "email": "jane@acme.example", - "errors": 0, - "exists_on": [ - "twitter" - ], - "partial": false, - "sites": [ - { - "domain": "twitter.com", - "exists": true, - "method": "register", - "name": "twitter", - "rate_limited": false - }, - { - "domain": "spotify.com", - "exists": false, - "method": "register", - "name": "spotify", - "rate_limited": true - } - ], - "total_checked": 2 - }, - "status": 200 -} diff --git a/apps/backend-go/internal/browserclient/testdata/recorded/stats.json b/apps/backend-go/internal/browserclient/testdata/recorded/stats.json index eae106b7f..6d689b32b 100644 --- a/apps/backend-go/internal/browserclient/testdata/recorded/stats.json +++ b/apps/backend-go/internal/browserclient/testdata/recorded/stats.json @@ -16,11 +16,6 @@ "errors": {}, "ok": 2 }, - "holehe": { - "calls": 1, - "errors": {}, - "ok": 1 - }, "search": { "calls": 2, "errors": {}, diff --git a/apps/backend-go/internal/browserclient/types.go b/apps/backend-go/internal/browserclient/types.go index a9f8bc2bd..d3a265f82 100644 --- a/apps/backend-go/internal/browserclient/types.go +++ b/apps/backend-go/internal/browserclient/types.go @@ -420,45 +420,6 @@ type ExtractResult struct { ElapsedMS int `json:"elapsed_ms"` } -// ── holehe ────────────────────────────────────────────────────────── - -// HoleheRequest is POST /v1/holehe. MaxModules zero means 50 (legacy). -// Timeout zero means 60 s. -type HoleheRequest struct { - Email string `json:"email"` - MaxModules int `json:"max_modules,omitempty"` - UpstreamProxy ProxyURL `json:"upstream_proxy,omitempty"` - Timeout time.Duration `json:"-"` -} - -func (r HoleheRequest) MarshalJSON() ([]byte, error) { - type alias HoleheRequest - return json.Marshal(struct { - alias - TimeoutS float64 `json:"timeout_s,omitempty"` - }{alias(r), secs(r.Timeout)}) -} - -type HoleheSite struct { - Name string `json:"name"` - Domain string `json:"domain"` - Method string `json:"method"` - Exists bool `json:"exists"` - RateLimited bool `json:"rate_limited"` -} - -// HoleheResult: raw per-site results in module order. The verdict mapping -// (legacy is_real = len(ExistsOn) >= 2, email_verify, confidence) is Go's. -type HoleheResult struct { - Email string `json:"email"` - Sites []HoleheSite `json:"sites"` - ExistsOn []string `json:"exists_on"` - TotalChecked int `json:"total_checked"` - Errors int `json:"errors"` - Partial bool `json:"partial"` - ElapsedMS int `json:"elapsed_ms"` -} - // ── staffspy ──────────────────────────────────────────────────────── // StaffSpyRequest is POST /v1/staffspy (503 engine_unavailable unless the diff --git a/apps/backend-go/internal/providers/scrape/library.go b/apps/backend-go/internal/providers/scrape/library.go index e26e8293e..2b36e1088 100644 --- a/apps/backend-go/internal/providers/scrape/library.go +++ b/apps/backend-go/internal/providers/scrape/library.go @@ -16,50 +16,6 @@ import ( "github.com/debpalash/OpenGTM/apps/backend-go/internal/providers" ) -// ── holehe_verify.py (the holehe library runs as the browser op) ─────────── - -type holehe struct{ gated } - -func (holehe) Name() string { return "holehe" } - -func (holehe) Enrich(ctx context.Context, _ *http.Client, _ func() (string, error), in providers.Input) (providers.Output, error) { - email := ls(in.Lead, "email") - if email == "" || !strings.Contains(email, "@") { - return fail(in, "No email to verify"), nil - } - b := current().Browser - if b == nil { - return fail(in, errNoWeb), nil - } - res, err := b.Holehe(ctx, browserclient.HoleheRequest{Email: email}) - if err != nil { - if cerr := ctx.Err(); cerr != nil { - return providers.Output{}, cerr - } - // Deviation: Python reported the library exception text; the browser - // service returns fixed error codes only. - return fail(in, "holehe unavailable: "+browserCode(err)), nil - } - exists := res.ExistsOn - f := providers.NewObj() - f.Set("email", email) - var conf float64 - switch { - case len(exists) >= 2: - f.Set("email_verify", "verified_active") - f.Set("email_presence", strings.Join(exists[:min(5, len(exists))], ", ")) - conf = math.Min(0.95, 0.70+float64(len(exists))*0.05) - case len(exists) == 1: - f.Set("email_verify", "likely_valid") - f.Set("email_presence", strings.Join(exists, ", ")) - conf = 0.65 - default: - f.Set("email_verify", "no_presence") - conf = 0.30 - } - return result(in, f, conf, ""), nil -} - func browserCode(err error) string { var be *browserclient.Error if errors.As(err, &be) && be.Code != "" { diff --git a/apps/backend-go/internal/providers/scrape/register.go b/apps/backend-go/internal/providers/scrape/register.go index bde951794..20c36dba1 100644 --- a/apps/backend-go/internal/providers/scrape/register.go +++ b/apps/backend-go/internal/providers/scrape/register.go @@ -15,7 +15,6 @@ func init() { providers.Register(ddgEmail{}) providers.Register(decisionMaker{}) providers.Register(JobSpy{}) - providers.Register(holehe{}) providers.Register(staffSpy{}) providers.Register(mailScout{}) providers.Register(companySizeHeuristic{}) diff --git a/apps/backend-go/internal/providers/scrape/scrape.go b/apps/backend-go/internal/providers/scrape/scrape.go index 36bf8f304..1f8a01d98 100644 --- a/apps/backend-go/internal/providers/scrape/scrape.go +++ b/apps/backend-go/internal/providers/scrape/scrape.go @@ -59,10 +59,9 @@ type Mail interface { } // Browser is the part of the browser-service client the providers call -// directly (holehe, staffspy, the person_intel_deep page render). +// directly (staffspy, the person_intel_deep page render). type Browser interface { Render(ctx context.Context, req browserclient.RenderRequest) (browserclient.RenderResult, error) - Holehe(ctx context.Context, req browserclient.HoleheRequest) (browserclient.HoleheResult, error) StaffSpy(ctx context.Context, req browserclient.StaffSpyRequest) (browserclient.StaffSpyResult, error) } diff --git a/apps/backend-go/internal/providers/scrape/testdata/vectors.json b/apps/backend-go/internal/providers/scrape/testdata/vectors.json index b08e2487a..1fbdb12a6 100644 --- a/apps/backend-go/internal/providers/scrape/testdata/vectors.json +++ b/apps/backend-go/internal/providers/scrape/testdata/vectors.json @@ -2433,121 +2433,6 @@ "license": "unknown" } }, - { - "name": "hh_no_email", - "provider": "holehe", - "input": { - "email": "nope" - }, - "calls": [], - "output": { - "provider": "holehe", - "success": false, - "fields": {}, - "confidence": 0.0, - "error": "No email to verify", - "billing_evidence": null, - "license": "unknown" - } - }, - { - "name": "hh_real", - "provider": "holehe", - "input": { - "email": "a@b.com" - }, - "holehe": { - "exists_on": [ - "twitter", - "spotify", - "adobe", - "imgur", - "x", - "y" - ], - "total_checked": 50 - }, - "calls": [ - [ - "holehe", - "a@b.com" - ] - ], - "output": { - "provider": "holehe", - "success": true, - "fields": { - "email": "a@b.com", - "email_verify": "verified_active", - "email_presence": "twitter, spotify, adobe, imgur, x" - }, - "confidence": 0.95, - "error": "", - "billing_evidence": null, - "license": "unknown" - } - }, - { - "name": "hh_one", - "provider": "holehe", - "input": { - "email": "a@b.com" - }, - "holehe": { - "exists_on": [ - "github" - ], - "total_checked": 48 - }, - "calls": [ - [ - "holehe", - "a@b.com" - ] - ], - "output": { - "provider": "holehe", - "success": true, - "fields": { - "email": "a@b.com", - "email_verify": "likely_valid", - "email_presence": "github" - }, - "confidence": 0.65, - "error": "", - "billing_evidence": null, - "license": "unknown" - } - }, - { - "name": "hh_zero", - "provider": "holehe", - "input": { - "email": "a@b.com" - }, - "holehe": { - "exists_on": [], - "total_checked": 50 - }, - "calls": [ - [ - "holehe", - "a@b.com" - ] - ], - "output": { - "provider": "holehe", - "success": true, - "fields": { - "email": "a@b.com", - "email_verify": "no_presence" - }, - "confidence": 0.3, - "error": "", - "billing_evidence": null, - "license": "unknown" - } - }, { "name": "ss_no_company", "provider": "staffspy", diff --git a/apps/backend-go/internal/providers/scrape/unit_test.go b/apps/backend-go/internal/providers/scrape/unit_test.go index f0f677199..e5ae9f016 100644 --- a/apps/backend-go/internal/providers/scrape/unit_test.go +++ b/apps/backend-go/internal/providers/scrape/unit_test.go @@ -23,9 +23,6 @@ func (f *renderFake) Render(_ context.Context, r browserclient.RenderRequest) (b f.reqs = append(f.reqs, r) return f.fn(r) } -func (f *renderFake) Holehe(context.Context, browserclient.HoleheRequest) (browserclient.HoleheResult, error) { - return browserclient.HoleheResult{}, &browserclient.Error{Code: "busy"} -} func (f *renderFake) StaffSpy(context.Context, browserclient.StaffSpyRequest) (browserclient.StaffSpyResult, error) { return browserclient.StaffSpyResult{}, nil } @@ -100,7 +97,7 @@ func TestPersonIntelDeepFailures(t *testing.T) { func TestScrapeProvidersAreGated(t *testing.T) { for _, n := range []string{"website_scraper", "deep_scraper", "company_intel", "local_business", "crosslinked", "email_harvester", "facebook_pages", "social_finder", "ddg_company", "ddg_email", "decision_maker", "jobspy", - "holehe", "staffspy", "mailscout", "company_size_heuristic", "lead_scorer"} { + "staffspy", "mailscout", "company_size_heuristic", "lead_scorer"} { p, ok := providers.Get(n) if !ok { t.Fatalf("%s not registered", n) @@ -213,10 +210,6 @@ func TestStaffSpyOffWithoutOptIn(t *testing.T) { if out, _ := (staffSpy{}).Enrich(context.Background(), nil, nil, in); out.Error != staffspyUnavailable { t.Errorf("no li_at: %q", out.Error) } - Configure(Deps{Browser: &renderFake{}}) - if out, _ := (holehe{}).Enrich(context.Background(), nil, nil, providers.Input{Lead: map[string]any{"email": "a@b.co"}}); out.Error != "holehe unavailable: busy" { - t.Errorf("holehe: %q", out.Error) - } } func TestNoSeamIsAnErrorOutput(t *testing.T) { diff --git a/apps/backend-go/internal/providers/scrape/vectors_test.go b/apps/backend-go/internal/providers/scrape/vectors_test.go index df34f734d..a0f81435e 100644 --- a/apps/backend-go/internal/providers/scrape/vectors_test.go +++ b/apps/backend-go/internal/providers/scrape/vectors_test.go @@ -48,10 +48,6 @@ type scenario struct { Pick mailAns `json:"pick"` Verify mailAns `json:"verify"` } `json:"mail"` - Holehe struct { - ExistsOn []string `json:"exists_on"` - TotalChecked int `json:"total_checked"` - } `json:"holehe"` StaffSpy struct { Rows []browserclient.StaffRow `json:"rows"` } `json:"staffspy"` @@ -173,11 +169,6 @@ func (w *fakeWorld) Render(context.Context, browserclient.RenderRequest) (browse return browserclient.RenderResult{}, errors.New("render not scripted") } -func (w *fakeWorld) Holehe(_ context.Context, r browserclient.HoleheRequest) (browserclient.HoleheResult, error) { - w.log("holehe", r.Email) - return browserclient.HoleheResult{Email: r.Email, ExistsOn: w.sc.Holehe.ExistsOn, TotalChecked: w.sc.Holehe.TotalChecked}, nil -} - func (w *fakeWorld) StaffSpy(_ context.Context, r browserclient.StaffSpyRequest) (browserclient.StaffSpyResult, error) { w.log("staffspy", r.CompanyName, r.Location, r.MaxResults) return browserclient.StaffSpyResult{CompanyName: r.CompanyName, Staff: w.sc.StaffSpy.Rows}, nil diff --git a/apps/backend-go/internal/providers/testdata/python_provider_inventory.json b/apps/backend-go/internal/providers/testdata/python_provider_inventory.json index 2795b69cd..8041aec2a 100644 --- a/apps/backend-go/internal/providers/testdata/python_provider_inventory.json +++ b/apps/backend-go/internal/providers/testdata/python_provider_inventory.json @@ -483,36 +483,6 @@ "sha256": "3867d40be1587fe152bdbd718b46940b480849fc30fd3df10ed3aca589dd5207", "registered_in_workbook_registry": true }, - { - "module": "holehe_verify", - "classes": [ - { - "class": "HoleheProvider", - "bases": [ - "EnrichmentProvider" - ], - "name": "holehe", - "capabilities": [ - "email_verify" - ], - "cost_per_lookup": null, - "default_confidence": 0.85 - } - ], - "imports_top_level": [], - "imports_nested": [ - "holehe", - "httpx" - ], - "non_http_third_party": [ - "holehe" - ], - "uses_httpx": true, - "uses_smtp_or_dns": false, - "line_count": 144, - "sha256": "b93b5d864b323c56316b235f24864b766f00a0fa06aeb0e6f26085e0612a3583", - "registered_in_workbook_registry": true - }, { "module": "hunter_io", "classes": [ diff --git a/apps/backend-go/internal/vendorcatalog/native_costs.go b/apps/backend-go/internal/vendorcatalog/native_costs.go index b13b01516..24f81eea0 100644 --- a/apps/backend-go/internal/vendorcatalog/native_costs.go +++ b/apps/backend-go/internal/vendorcatalog/native_costs.go @@ -15,7 +15,6 @@ var nativeFreeProviders = map[string]bool{ "email_harvester": true, "facebook_pages": true, "gleif": true, - "holehe": true, "ipinfo": true, "jobspy": true, "jsonld_firmographics": true, diff --git a/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json b/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json index 7106a97e0..ae0bfb86c 100644 --- a/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json +++ b/apps/backend-go/internal/vendorcatalog/testdata/catalog_vectors.json @@ -156,13 +156,6 @@ ], "default_confidence": 0.9 }, - "holehe": { - "cost_per_lookup": 0.0, - "capabilities": [ - "email_verify" - ], - "default_confidence": 0.85 - }, "hunter_io": { "cost_per_lookup": 0.0, "capabilities": [ @@ -807,22 +800,6 @@ "has_known_cost": true, "is_byok": true }, - { - "name": "holehe", - "research_flag": false, - "base_cost": 0.0, - "is_paid": false, - "has_known_cost": true, - "is_byok": true - }, - { - "name": "holehe", - "research_flag": true, - "base_cost": 0.0, - "is_paid": false, - "has_known_cost": true, - "is_byok": true - }, { "name": "hunter_io", "research_flag": false, diff --git a/apps/backend-go/internal/wbregistry/registry.json b/apps/backend-go/internal/wbregistry/registry.json index a8178ea10..2d81d9e66 100644 --- a/apps/backend-go/internal/wbregistry/registry.json +++ b/apps/backend-go/internal/wbregistry/registry.json @@ -627,23 +627,6 @@ "maturity": "beta", "certification": null }, - { - "name": "holehe", - "capabilities": [ - "email_verify" - ], - "confidence": 0.85, - "required_checks": [ - "external_read", - "failure_recovery", - "normalization", - "provenance", - "quality_sample", - "tenant_isolation" - ], - "maturity": "beta", - "certification": null - }, { "name": "email_harvester", "capabilities": [ @@ -1443,8 +1426,7 @@ "type": "waterfall", "target_field": "email_verify", "waterfall": [ - "mailscout", - "holehe" + "mailscout" ], "condition": "{email} != \"\"", "width": 160 diff --git a/apps/backend-go/internal/workbook/cell.go b/apps/backend-go/internal/workbook/cell.go index 447a44182..3cda3c239 100644 --- a/apps/backend-go/internal/workbook/cell.go +++ b/apps/backend-go/internal/workbook/cell.go @@ -579,6 +579,15 @@ loop: skipped = append(skipped, skippedProvider{name, "unknown_provider"}) continue } + // A stored chain can name a provider that was removed from this build + // (e.g. holehe): skip it, like any unknown provider, instead of failing + // the run or stopping the waterfall before the later providers. A priced + // provider with no runner keeps its own path (released spend, cell fails). + if _, declarative := rs.p.Manifests[name]; !declarative && !rs.pl.paid(name) && !rs.registered(name) { + rs.log.Warn("provider not found, skipping", "provider", name) + skipped = append(skipped, skippedProvider{name, "unknown_provider"}) + continue + } reserved := rs.pl.paid(name) if !reserved && !rs.pl.knownCost(name) { resultError = "provider_price_unknown" diff --git a/apps/backend-go/internal/workbook/defaults.go b/apps/backend-go/internal/workbook/defaults.go index 6b01ad4eb..d36aa9a48 100644 --- a/apps/backend-go/internal/workbook/defaults.go +++ b/apps/backend-go/internal/workbook/defaults.go @@ -13,8 +13,8 @@ var defaultWaterfalls = map[string][]string{ "ddg_email", "mailscout", "hunter_io", "apollo_io", "snovio", "prospeo", }, - "email_confidence": {"mailscout", "holehe", "abstract_api", "debounce"}, - "email_verify": {"mailscout", "holehe", "abstract_api", "debounce"}, + "email_confidence": {"mailscout", "abstract_api", "debounce"}, + "email_verify": {"mailscout", "abstract_api", "debounce"}, "phone": { "deep_scraper", "jsonld_firmographics", "website_scraper", "local_business", "ddg_company", "facebook_pages", diff --git a/apps/backend-go/internal/workbook/provenance.go b/apps/backend-go/internal/workbook/provenance.go index 22fa902ab..9d0d246aa 100644 --- a/apps/backend-go/internal/workbook/provenance.go +++ b/apps/backend-go/internal/workbook/provenance.go @@ -28,7 +28,7 @@ var providerLicense = map[string]string{ "hunter_io": "proprietary-api", "apollo_io": "proprietary-api", "snovio": "proprietary-api", "prospeo": "proprietary-api", "people_data_labs": "proprietary-api", "clearbit": "proprietary-api", "abstract_api": "proprietary-api", "debounce": "proprietary-api", "deep_scraper": "scraped", "website_scraper": "scraped", "jsonld_firmographics": "scraped", "email_harvester": "scraped", - "ddg_email": "scraped", "ddg_company": "scraped", "mailscout": "scraped", "holehe": "scraped", "social_finder": "scraped", + "ddg_email": "scraped", "ddg_company": "scraped", "mailscout": "scraped", "social_finder": "scraped", "crosslinked": "scraped", "staffspy": "scraped", "decision_maker": "scraped", "local_business": "scraped", "facebook_pages": "scraped", "google_maps": "scraped", "tech_stack": "scraped", "company_intel": "scraped", "jobspy": "scraped", "ats_hiring": "scraped", "company_size_heuristic": "scraped", "lead_scorer": "scraped", diff --git a/apps/backend-go/internal/workbook/removed_provider_integration_test.go b/apps/backend-go/internal/workbook/removed_provider_integration_test.go new file mode 100644 index 000000000..27f1a5339 --- /dev/null +++ b/apps/backend-go/internal/workbook/removed_provider_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package workbook_test + +import ( + "context" + "testing" + "time" + + "github.com/debpalash/OpenGTM/apps/backend-go/internal/provcall" + "github.com/debpalash/OpenGTM/apps/backend-go/internal/providers" + "github.com/debpalash/OpenGTM/apps/backend-go/internal/workbook" +) + +// removedRunner serves only "ok"; every other name is neither a Go provider nor +// a manifest, like a provider that was removed from the build (holehe). +type removedRunner struct{ calls []string } + +func (r *removedRunner) Serves(name string) bool { return name == "ok" } + +func (r *removedRunner) Run(_ context.Context, _, name string, _ providers.Input, _ time.Duration) (providers.Output, error) { + r.calls = append(r.calls, name) + if name != "ok" { + return providers.Output{}, provcall.ErrUnavailable + } + f := providers.NewObj() + f.Set(target, "7") + return providers.Output{Extra: f, Confidence: 0.9}, nil +} + +// A stored waterfall that still names a removed provider must skip it and let +// the later providers run, not fail the run or stop the waterfall. +func TestStoredChainNamingRemovedProviderIsSkipped(t *testing.T) { + e := setup(t, wsA) + e.seed(wsA, "wb-gone", "["+colJSON("c1", "holehe", "ok")+"]", 1, 0) + raw := mutate(payloadFor(wsA, "wb-gone", map[string]float64{"holehe": 0, "ok": 0}, nil), func(p map[string]any) { + delete(p["go"].(map[string]any)["manifests"].(map[string]any), "holehe") + }) + r := &removedRunner{} + d := e.deps(newSim(t, nil), nil) + d.Runner = r + res, err := workbook.Run(e.ctx, d, e.claim(wsA, raw)) + if err != nil || res.Completed != 1 || res.Errors != 0 { + t.Fatalf("%+v %v", res, err) + } + if len(r.calls) != 1 || r.calls[0] != "ok" { + t.Fatalf("the removed provider must never be called: %v", r.calls) + } + if v := e.str(wsA, `SELECT value FROM workbook_enrichments WHERE column_id='c1'`); v != "7" { + t.Fatalf("value %q", v) + } +} diff --git a/apps/backend-go/internal/workbook/runner.go b/apps/backend-go/internal/workbook/runner.go index 48c9320c5..c89e5d434 100644 --- a/apps/backend-go/internal/workbook/runner.go +++ b/apps/backend-go/internal/workbook/runner.go @@ -105,6 +105,16 @@ func (rs *runState) serves(name string) bool { return true } +// registered reports whether name is a provider this build knows: a Go-native +// provider, or one the runner serves (a verified manifest). A name that is +// neither was removed or never existed. +func (rs *runState) registered(name string) bool { + if _, ok := providers.Get(name); ok { + return true + } + return rs.serves(name) +} + // precheck refuses a call the runner can already tell will not reach the network. func (rs *runState) precheck(ctx context.Context, name string, lead map[string]any) string { if pc, ok := rs.runner.(prechecker); ok { diff --git a/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json b/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json index 9e0be9c1b..790aa4243 100644 --- a/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json +++ b/apps/backend-go/internal/workbook/testdata/wbexec_vectors.json @@ -5819,7 +5819,6 @@ "ddg_email": "scraped", "ddg_company": "scraped", "mailscout": "scraped", - "holehe": "scraped", "social_finder": "scraped", "crosslinked": "scraped", "staffspy": "scraped", diff --git a/apps/backend-go/testdata/contract/meta/providers.json b/apps/backend-go/testdata/contract/meta/providers.json index aa861a7e2..e3c7c4d97 100644 --- a/apps/backend-go/testdata/contract/meta/providers.json +++ b/apps/backend-go/testdata/contract/meta/providers.json @@ -458,23 +458,6 @@ "tenant_isolation" ] }, - { - "capabilities": [ - "email_verify" - ], - "certification": null, - "confidence": 0.85, - "maturity": "beta", - "name": "holehe", - "required_checks": [ - "external_read", - "failure_recovery", - "normalization", - "provenance", - "quality_sample", - "tenant_isolation" - ] - }, { "capabilities": [ "email", diff --git a/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json b/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json index 3c2978ffc..dec8621a8 100644 --- a/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json +++ b/apps/backend-go/testdata/contract/meta/providers_no_ws_header.json @@ -457,23 +457,6 @@ "tenant_isolation" ] }, - { - "capabilities": [ - "email_verify" - ], - "certification": null, - "confidence": 0.85, - "maturity": "beta", - "name": "holehe", - "required_checks": [ - "external_read", - "failure_recovery", - "normalization", - "provenance", - "quality_sample", - "tenant_isolation" - ] - }, { "capabilities": [ "email", diff --git a/apps/browser-service/pyproject.toml b/apps/browser-service/pyproject.toml index d82389528..8f44c05f1 100644 --- a/apps/browser-service/pyproject.toml +++ b/apps/browser-service/pyproject.toml @@ -8,7 +8,6 @@ dependencies = [ "curl-cffi>=0.17.0", "ddgs>=9.16.1", "fastapi>=0.162.0", - "holehe>=1.61", "httpx>=0.28.1", "lxml>=6.1.3", "markdownify>=1.2.3", diff --git a/apps/browser-service/scripts/record_contract_fixtures.py b/apps/browser-service/scripts/record_contract_fixtures.py index fd67c3aa6..4a0a624a2 100644 --- a/apps/browser-service/scripts/record_contract_fixtures.py +++ b/apps/browser-service/scripts/record_contract_fixtures.py @@ -2,7 +2,7 @@ uv run python scripts/record_contract_fixtures.py ../backend-go/internal/browserclient/testdata/recorded -Runs the FastAPI app in-process with fake browser / ddgs / holehe / staffspy +Runs the FastAPI app in-process with fake browser / ddgs / staffspy backends (no network) and saves request + response pairs; extract runs the real trafilatura and markdownify. """ @@ -19,7 +19,7 @@ from browser_service.app import create_app from browser_service.config import Settings -from browser_service.scrapers import holehe_check, staffspy_roster +from browser_service.scrapers import staffspy_roster TOKEN = "r" * 40 ARTICLE = ( @@ -79,19 +79,6 @@ async def fake_search(deps, method, search, max_results, deadline, **opts): async def main(out: Path) -> None: - async def fn(email, client, o, _n="twitter"): - o.append({"name": _n, "domain": "twitter.com", "method": "register", "rateLimit": False, "exists": True}) - - async def fn2(email, client, o): - o.append({"name": "spotify", "domain": "spotify.com", "method": "register", "rateLimit": True, "exists": False}) - - holehe_check.load_modules = lambda: [fn, fn2] - - class C: - async def aclose(self): - pass - - holehe_check.make_client = lambda p, t: C() staffspy_roster.check_logged_in = lambda s: True staffspy_roster.scrape_staff = lambda s, c, t, loc, n: [ { @@ -109,7 +96,6 @@ async def aclose(self): "search_error": ("POST", "/v1/search", {"query": "fail", "max_results": 2}), "extract": ("POST", "/v1/extract", {"html": ARTICLE, "url": "https://acme.example/a"}), "extract_url": ("POST", "/v1/extract", {"url": "https://acme.example/a", "engine": "browser"}), - "holehe": ("POST", "/v1/holehe", {"email": "jane@acme.example"}), "staffspy": ( "POST", "/v1/staffspy", diff --git a/apps/browser-service/src/browser_service/app.py b/apps/browser-service/src/browser_service/app.py index d7393713b..5fd76799b 100644 --- a/apps/browser-service/src/browser_service/app.py +++ b/apps/browser-service/src/browser_service/app.py @@ -40,8 +40,6 @@ GoogleMapsResult, GoogleSearchRequest, Health, - HoleheRequest, - HoleheResult, JobBoardsRequest, JustDialPageRequest, JustDialPageResult, @@ -61,7 +59,7 @@ Stats, WebDirectoriesRequest, ) -from .scrapers import google_maps, holehe_check, justdial, search, staffspy_roster +from .scrapers import google_maps, justdial, search, staffspy_roster logger = logging.getLogger("browser_service.app") @@ -329,10 +327,6 @@ async def search_query(req: SearchRequest) -> SearchOut: async def extract_content(req: ExtractRequest) -> ExtractResult: return await run("extract", req.timeout_s, lambda d: extract.extract(deps, req, d)) - @v1.post("/holehe", response_model=HoleheResult) - async def holehe(req: HoleheRequest) -> HoleheResult: - return await run("holehe", req.timeout_s, lambda d: holehe_check.check(deps, req, d)) - @v1.post("/staffspy", response_model=StaffSpyResult) async def staffspy(req: StaffSpyRequest) -> StaffSpyResult: return await run("staffspy", req.timeout_s, lambda d: staffspy_roster.scrape(deps, req, d)) diff --git a/apps/browser-service/src/browser_service/models.py b/apps/browser-service/src/browser_service/models.py index 2cd44fb3b..da0e109ab 100644 --- a/apps/browser-service/src/browser_service/models.py +++ b/apps/browser-service/src/browser_service/models.py @@ -325,33 +325,6 @@ class ExtractResult(BaseModel): elapsed_ms: int -# ── holehe ─────────────────────────────────────────────────────────── - - -class HoleheRequest(_Upstream): - email: str = Field(min_length=3, max_length=254, pattern=r"^[^@\s]+@[^@\s]+\.[^@\s]+$") - max_modules: int = Field(default=50, ge=1, le=200, description="legacy checked the first 50 modules") - timeout_s: float = Field(default=60.0, ge=5.0, le=300.0) - - -class HoleheSite(BaseModel): - name: str - domain: str - method: str - exists: bool - rate_limited: bool - - -class HoleheResult(BaseModel): - email: str - sites: list[HoleheSite] - exists_on: list[str] - total_checked: int - errors: int = Field(description="modules that raised") - partial: bool - elapsed_ms: int - - # ── staffspy ───────────────────────────────────────────────────────── diff --git a/apps/browser-service/src/browser_service/scrapers/holehe_check.py b/apps/browser-service/src/browser_service/scrapers/holehe_check.py deleted file mode 100644 index 15dc89a23..000000000 --- a/apps/browser-service/src/browser_service/scrapers/holehe_check.py +++ /dev/null @@ -1,116 +0,0 @@ -"""POST /v1/holehe: which sites an email is registered on (legacy -leadgen/enrichment/providers/holehe_verify.py). - -Same module set and order as legacy (pkgutil walk of holehe.modules, first 50 -by default) and the same concurrency (10) and per-request timeout (10 s). Every -module's traffic goes through the egress proxy. The confidence / verdict -mapping (is_real, email_verify) stays in Go; this returns raw per-site results. - -Deviation (legacy bug, fixed): legacy passed a dict as holehe's ``out`` list, -so every module raised on ``out.append`` and no site was ever reported. -""" - -from __future__ import annotations - -import asyncio -import importlib -import logging -import pkgutil -from functools import cache -from typing import Any - -from ..engines import Deadline, Deps -from ..models import HoleheRequest, HoleheResult, HoleheSite - -logger = logging.getLogger("browser_service.scrapers.holehe") - -CONCURRENCY = 10 -CLIENT_TIMEOUT_S = 10 - - -@cache -def _modules() -> tuple: - import holehe.modules - - out = [] - for _imp, modname, ispkg in pkgutil.walk_packages(holehe.modules.__path__, holehe.modules.__name__ + "."): - if ispkg: - continue - try: - mod = importlib.import_module(modname) - except Exception: # noqa: S112 - a broken module is skipped, as in legacy - continue - fn = getattr(mod, modname.rsplit(".", 1)[-1], None) - if fn is not None: - out.append(fn) - return tuple(out) - - -def load_modules() -> list: - return list(_modules()) - - -def make_client(proxy_url: str, timeout: float) -> Any: - import httpx - - return httpx.AsyncClient(proxy=proxy_url, timeout=timeout, trust_env=False, follow_redirects=False) - - -async def check(deps: Deps, req: HoleheRequest, deadline: Deadline) -> HoleheResult: - modules = load_modules()[: req.max_modules] - session = deps.proxy.open_session(deps.settings.max_download_bytes, upstream=req.upstream()) - client = make_client(deps.proxy.proxy_url(session), CLIENT_TIMEOUT_S) - sem = asyncio.Semaphore(CONCURRENCY) - errors = 0 - - async def one(fn) -> dict | None: - nonlocal errors - async with sem: - out: list = [] - try: - await fn(req.email, client, out) - except Exception: - errors += 1 - return None - return out[0] if out and isinstance(out[0], dict) else None - - tasks = [asyncio.ensure_future(one(fn)) for fn in modules] - partial = False - try: - done, pending = await asyncio.wait(tasks, timeout=max(0.5, deadline.left() - 1.0)) if tasks else (set(), set()) - if pending: - partial = True - for t in pending: - t.cancel() - await asyncio.gather(*pending, return_exceptions=True) - finally: - try: - await client.aclose() - finally: - deps.proxy.close_session(session) - - sites: list[HoleheSite] = [] - for t in tasks: # module order, as legacy gathered them - if t.cancelled() or t.exception() is not None: - continue - r = t.result() - if r is None: - continue - sites.append( - HoleheSite( - name=str(r.get("name", "unknown"))[:100], - domain=str(r.get("domain") or "")[:253], - method=str(r.get("method") or "")[:50], - exists=r.get("exists") is True, - rate_limited=bool(r.get("rateLimit")), - ) - ) - return HoleheResult( - email=req.email, - sites=sites, - exists_on=[s.name for s in sites if s.exists], - total_checked=len(sites), - errors=errors, - partial=partial, - elapsed_ms=deadline.elapsed_ms(), - ) diff --git a/apps/browser-service/tests/test_ops.py b/apps/browser-service/tests/test_ops.py index 1acac55c4..adf311157 100644 --- a/apps/browser-service/tests/test_ops.py +++ b/apps/browser-service/tests/test_ops.py @@ -1,8 +1,7 @@ """Ops added for the Go web layer (plan 15 WP-WEB): generic search, extract, -holehe, staffspy, stats, per-request upstream proxies and the stealth Maps +staffspy, stats, per-request upstream proxies and the stealth Maps browser.""" -import asyncio import sys import types @@ -11,7 +10,7 @@ from browser_service.config import ConfigError, Settings from browser_service.egress import UpstreamProxy -from browser_service.scrapers import holehe_check, search, staffspy_roster +from browser_service.scrapers import search, staffspy_roster ARTICLE = ( "