From 3bf3991151accbc1d63bfaec456a7449490a35ce Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sat, 25 Apr 2026 21:01:08 -0700 Subject: [PATCH 1/4] Rename sf-hamilton to apache-hamilton across docs, examples, and source Update all references to sf-hamilton package names to use the apache-hamilton prefix. Covers error messages, install instructions, requirements.txt files, notebooks, CI workflows, VSCode extension, and UI components. Import names are unchanged. The pyproject.toml extras for lsp/sdk/ui still point to sf-hamilton-* packages until stable apache-hamilton-* versions are published to PyPI. --- .claude-plugin/skills/observability/SKILL.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 4 +- .../workflows/contrib-auto-build-publish.yml | 2 +- .../workflows/hamilton-ui-build-and-push.yml | 4 +- README.md | 16 +- contrib/README.md | 4 +- contrib/docs/compile_docs.py | 4 +- contrib/docs/docs/README.md | 4 +- contrib/docs/src/components/example_code.tsx | 2 +- .../sphinx_doc_chunking/requirements.txt | 8 +- .../air_quality_analysis/requirements.txt | 2 +- .../customize_embeddings/requirements.txt | 2 +- .../user/skrawcz/fine_tuning/requirements.txt | 2 +- .../user/zilto/lancedb_vdb/requirements.txt | 2 +- .../user/zilto/webscraper/requirements.txt | 2 +- .../zilto/xgboost_optuna/requirements.txt | 2 +- contrib/requirements.txt | 2 +- contrib/scripts/version_check.py | 2 +- .../walkthrough/3_pythonDependencies.md | 4 +- dev_tools/vscode_extension/src/extension.ts | 4 +- .../using-within-your-etl-system.rst | 2 +- docs/concepts/function-modifiers.rst | 4 +- docs/conf.py | 2 +- docs/get-started/install.md | 8 +- docs/get-started/your-first-dataflow.rst | 2 +- docs/hamilton-ui/ui.rst | 4 +- docs/hamilton-vscode/language_server.rst | 4 +- docs/how-tos/cli-reference.md | 2 +- docs/how-tos/pre-commit-hooks.md | 2 +- docs/how-tos/use-in-jupyter-notebook.md | 2 +- docs/main.md | 4 +- docs/reference/decorators/with_columns.rst | 2 +- docs/reference/lifecycle-hooks/Narwhals.rst | 4 +- .../reference/lifecycle-hooks/ProgressBar.rst | 2 +- .../lifecycle-hooks/RichProgressBar.rst | 2 +- .../lifecycle-hooks/SlackNotifierHook.rst | 2 +- .../GraphRAG/ingest_notebook.ipynb | 2 +- .../LLM_Workflows/GraphRAG/requirements.txt | 2 +- .../LLM_Workflows/NER_Example/notebook.ipynb | 2 +- .../NER_Example/requirements.txt | 2 +- .../requirements.txt | 2 +- .../simple_pipeline.ipynb | 4 +- .../image_telephone/notebook.ipynb | 2 +- .../image_telephone/requirements.txt | 4 +- .../knowledge_retrieval/requirements.txt | 2 +- .../langchain_comparison/requirements.txt | 2 +- .../llm_logging/requirements.txt | 2 +- .../modular_llm_stack/requirements.txt | 4 +- .../neo4j_graph_rag/requirements.txt | 2 +- .../observability_openllmetry/notebook.ipynb | 2 +- .../requirements.txt | 2 +- .../pdf_summarizer/backend/requirements.txt | 2 +- .../run_on_spark/requirements.txt | 2 +- .../pdf_summarizer/run_on_spark/run.ipynb | 2 +- .../backend/ingestion.py | 2 +- .../backend/requirements.txt | 2 +- .../scraping_and_chunking/notebook.ipynb | 180 +++++++++--------- .../scraping_and_chunking/requirements.txt | 6 +- .../spark/notebook.ipynb | 2 +- .../spark/requirements.txt | 6 +- examples/README.md | 6 +- examples/airflow/requirements.txt | 4 +- examples/async/README.md | 2 +- examples/async/requirements.txt | 2 +- examples/aws/glue/README.md | 2 +- examples/aws/lambda/requirements.txt | 2 +- .../aws/sagemaker/container/requirements.txt | 2 +- examples/aws/sagemaker/notebook.ipynb | 2 +- examples/caching/in_memory_tutorial.ipynb | 2 +- examples/caching/materializer_tutorial.ipynb | 2 +- examples/caching/requirements.txt | 2 +- examples/caching/tutorial.ipynb | 2 +- examples/caching_nodes/caching.ipynb | 2 +- .../caching_graph_adapter/caching_nodes.ipynb | 2 +- .../caching_graph_adapter/requirements.txt | 2 +- .../diskcache_adapter/notebook.ipynb | 2 +- .../diskcache_adapter/requirements.txt | 2 +- examples/cli/README.md | 2 +- examples/cli/requirements.txt | 2 +- examples/contrib/README.md | 2 +- examples/contrib/notebook.ipynb | 8 +- .../dagworks-translate_to_hamilton.ipynb | 2 +- examples/contrib/requirements.txt | 4 +- examples/dagster/hamilton_code/notebook.ipynb | 2 +- examples/dagster/requirements.txt | 2 +- .../community_demo/demo_day_notebook.ipynb | 4 +- examples/dask/community_demo/requirements.txt | 2 +- examples/dask/hello_world/README.md | 2 +- examples/dask/hello_world/notebook.ipynb | 4 +- examples/dask/hello_world/run.py | 4 +- examples/dask/hello_world/run_with_delayed.py | 4 +- .../run_with_delayed_and_dask_objects.py | 4 +- examples/dask/requirements.txt | 2 +- examples/data_loaders/data_loaders.ipynb | 4 +- .../pandera/requirements-dask.txt | 4 +- .../data_quality/pandera/requirements-ray.txt | 4 +- .../pandera/requirements-spark.txt | 4 +- .../data_quality/pandera/requirements.txt | 4 +- examples/data_quality/pandera/run.py | 2 +- examples/data_quality/pandera/run_dask.py | 2 +- examples/data_quality/pandera/run_ray.py | 2 +- examples/data_quality/pandera/run_spark.py | 2 +- .../data_quality/simple/requirements-dask.txt | 4 +- .../data_quality/simple/requirements-ray.txt | 4 +- .../simple/requirements-spark.txt | 4 +- examples/data_quality/simple/requirements.txt | 4 +- examples/data_quality/simple/run.py | 2 +- examples/data_quality/simple/run_dask.py | 2 +- examples/data_quality/simple/run_ray.py | 2 +- examples/data_quality/simple/run_spark.py | 2 +- examples/dbt/models/train_and_infer.py | 2 +- examples/dbt/requirements.txt | 2 +- examples/decoupling_io/requirements.txt | 2 +- examples/dlt/dlt_plugin.ipynb | 2 +- examples/dlt/notebook.ipynb | 2 +- examples/dlt/requirements.txt | 2 +- .../due_date_probabilities/notebook.ipynb | 2 +- examples/experiment_management/README.md | 2 +- examples/experiment_management/notebook.ipynb | 2 +- .../experiment_management/requirements.txt | 2 +- .../feature_repo/retrieval.ipynb | 2 +- examples/feast/requirements.txt | 2 +- .../scenario_1/requirements.txt | 2 +- .../scenario_2/requirements.txt | 2 +- .../contexts/streaming.py | 2 +- .../requirements.txt | 2 +- .../mpg-translation/MPGSimple.ipynb | 2 +- .../MPGSimpleAdvancedTarget.ipynb | 2 +- .../mpg-translation/MPGSimpleTarget.ipynb | 2 +- .../ProceduralMPGNotebook.ipynb | 2 +- .../mpg-translation/requirements.txt | 2 +- examples/hamilton_ui/README.md | 2 +- examples/hamilton_ui/notebook.ipynb | 2 +- examples/hamilton_ui/requirements.txt | 2 +- examples/hello_world/my_notebook.ipynb | 4 +- examples/hello_world/my_script.py | 2 +- examples/hello_world/requirements.txt | 2 +- .../ibis/feature_engineering/requirements.txt | 2 +- examples/ibis/jaffle_shop/requirements.txt | 2 +- examples/ibisml/requirements.txt | 2 +- examples/jupyter_notebook_magic/example.ipynb | 2 +- .../hamilton-code/notebooks/interactive.ipynb | 2 +- examples/kedro/hamilton-code/requirements.txt | 2 +- .../kedro-plugin/kedro_to_hamilton.ipynb | 2 +- examples/lineage/lineage_snippets.ipynb | 2 +- examples/lineage/requirements.txt | 2 +- .../notebook.ipynb | 2 +- .../requirements.txt | 2 +- examples/materialization/notebook.ipynb | 2 +- examples/materialization/requirements.txt | 2 +- examples/mlflow/requirements.txt | 2 +- examples/mlflow/tutorial.ipynb | 2 +- .../modular_example/notebook.ipynb | 2 +- .../Hamilton_for_ML_dataflows.ipynb | 4 +- .../scikit-learn/requirements.txt | 2 +- examples/model_examples/scikit-learn/run.py | 2 +- .../Hamilton-TimeSeriesmodel.ipynb | 2 +- .../time-series/requirements.txt | 2 +- .../notebook.ipynb | 2 +- examples/narwhals/notebook.ipynb | 2 +- examples/narwhals/requirements.txt | 2 +- .../hamilton_notebook.ipynb | 4 +- .../air-quality-analysis/requirements.txt | 2 +- examples/openlineage/notebook.ipynb | 2 +- examples/openlineage/requirements.txt | 2 +- .../pandas/materialization/notebook.ipynb | 2 +- .../pandas/materialization/requirements.txt | 2 +- .../pandas/split-apply-combine/my_wrapper.py | 2 +- .../pandas/split-apply-combine/notebook.ipynb | 6 +- .../split-apply-combine/requirements.txt | 2 +- examples/pandas/with_columns/notebook.ipynb | 2 +- .../file_processing/notebook.ipynb | 2 +- .../file_processing/requirements.txt | 2 +- .../lazy_threadpool_execution/notebook.ipynb | 2 +- .../requirements.txt | 6 +- .../parallelism/star_counting/notebook.ipynb | 2 +- examples/people_data_labs/notebook.ipynb | 2 +- examples/people_data_labs/requirements.txt | 2 +- examples/plotly/notebook.ipynb | 2 +- examples/plotly/requirements.txt | 2 +- examples/polars/README.md | 2 +- examples/polars/lazyframe/README.md | 2 +- examples/polars/lazyframe/my_script.py | 2 +- examples/polars/lazyframe/requirements.txt | 2 +- .../polars/materialization/notebook.ipynb | 2 +- .../polars/materialization/requirements.txt | 2 +- examples/polars/my_script.py | 2 +- examples/polars/notebook.ipynb | 4 +- examples/polars/requirements.txt | 2 +- examples/polars/with_columns/notebook.ipynb | 2 +- .../prefect/hamilton_prefect_notebook.ipynb | 2 +- examples/prefect/requirements.txt | 2 +- examples/ray/hello_world/README.md | 2 +- examples/ray/hello_world/notebook.ipynb | 4 +- examples/ray/hello_world/run.py | 2 +- .../hamilton_notebook.ipynb | 2 +- .../ray_Hamilton_UI_tracking/requirements.txt | 2 +- examples/ray/requirements.txt | 2 +- .../reusing_functions/reusing_functions.ipynb | 4 +- examples/reverse_etl/notebook.ipynb | 2 +- examples/reverse_etl/requirements.txt | 2 +- .../hamilton_notebook.ipynb | 2 +- .../transformer/hamilton_notebook.ipynb | 4 +- .../scikit-learn/transformer/requirements.txt | 2 +- examples/slack/notebook.ipynb | 2 +- examples/slack/requirements.txt | 2 +- .../snowflake/hamilton_ui/requirements.txt | 2 +- examples/spark/pandas_on_spark/README.md | 2 +- examples/spark/pandas_on_spark/run.py | 2 +- examples/spark/pyspark/notebook.ipynb | 2 +- .../example_usage.ipynb | 2 +- examples/spark/pyspark_udfs/README.md | 2 +- examples/spark/requirements.txt | 2 +- .../world_of_warcraft__pandas.ipynb | 6 +- .../world_of_warcraft__spark_v1.ipynb | 2 +- .../world_of_warcraft__spark_v2.ipynb | 2 +- examples/streamlit/requirements.txt | 2 +- examples/vaex/notebook.ipynb | 2 +- examples/vaex/requirements.txt | 2 +- examples/validate_examples.py | 4 +- .../static_validator/notebook.ipynb | 2 +- hamilton/cli/__main__.py | 2 +- hamilton/contrib/__init__.py | 6 +- hamilton/data_quality/default_validators.py | 2 +- hamilton/graph.py | 4 +- hamilton/plugins/h_dask.py | 2 +- hamilton/plugins/h_experiments/__main__.py | 2 +- hamilton/plugins/h_opentelemetry.py | 2 +- hamilton/plugins/h_ray.py | 2 +- hamilton/plugins/h_rich.py | 2 +- hamilton/plugins/h_slack.py | 2 +- hamilton/plugins/h_spark.py | 2 +- hamilton/plugins/h_tqdm.py | 2 +- hamilton/plugins/pydantic_extensions.py | 4 +- scripts/find_hanging_tests.sh | 85 +++++++++ ui/README.md | 6 +- ui/backend/pyproject.toml | 3 +- ui/backend/server/requirements-base.txt | 2 +- .../Project/ProjectLogInstructions.tsx | 2 +- ui/sdk/README.md | 2 +- .../data_processing/template_data.json | 2 +- .../templates/hello_world/template_data.json | 2 +- .../machine_learning/template_data.json | 2 +- .../template_data.json | 2 +- writeups/data_quality.md | 2 +- writeups/developer_setup.md | 6 +- 246 files changed, 494 insertions(+), 410 deletions(-) create mode 100755 scripts/find_hanging_tests.sh diff --git a/.claude-plugin/skills/observability/SKILL.md b/.claude-plugin/skills/observability/SKILL.md index fa430aa24..56073e2eb 100644 --- a/.claude-plugin/skills/observability/SKILL.md +++ b/.claude-plugin/skills/observability/SKILL.md @@ -26,7 +26,7 @@ Hamilton UI is a web-based dashboard for: ### 1. Install Hamilton with UI Support ```bash -pip install "sf-hamilton[sdk,ui]" +pip install "apache-hamilton[sdk,ui]" ``` ### 2. Start the Hamilton UI diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f93a3041f..048d2f058 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,8 +2,8 @@ --- PR TEMPLATE INSTRUCTIONS (1) --- -Looking to submit a Apache Hamilton Dataflow to the sf-hamilton-contrib module? If so go the the `Preview` tab and select the appropriate sub-template: -* [sf-hamilton-contrib template](?expand=1&template=HAMILTON_CONTRIB_PR_TEMPLATE.md) +Looking to submit a Apache Hamilton Dataflow to the apache-hamilton-contrib module? If so go the the `Preview` tab and select the appropriate sub-template: +* [apache-hamilton-contrib template](?expand=1&template=HAMILTON_CONTRIB_PR_TEMPLATE.md) Else, if not, please remove this block of text. diff --git a/.github/workflows/contrib-auto-build-publish.yml b/.github/workflows/contrib-auto-build-publish.yml index 8dc62c7e5..e1a98f7c2 100644 --- a/.github/workflows/contrib-auto-build-publish.yml +++ b/.github/workflows/contrib-auto-build-publish.yml @@ -11,7 +11,7 @@ jobs: publish-to-pypi: runs-on: ubuntu-latest # Specifying a GitHub environment is optional, but strongly encouraged - environment: sf-hamilton-contrib-publishing-environment + environment: apache-hamilton-contrib-publishing-environment permissions: # IMPORTANT: this permission is mandatory for trusted publishing id-token: write diff --git a/.github/workflows/hamilton-ui-build-and-push.yml b/.github/workflows/hamilton-ui-build-and-push.yml index 7a642f06e..abb77c40d 100644 --- a/.github/workflows/hamilton-ui-build-and-push.yml +++ b/.github/workflows/hamilton-ui-build-and-push.yml @@ -22,10 +22,10 @@ jobs: - name: Install jq run: sudo apt-get update && sudo apt-get install -y jq - - name: Check latest version of sf-hamilton-ui + - name: Check latest version of apache-hamilton-ui id: check_version run: | - response=$(curl -s https://pypi.org/pypi/sf-hamilton-ui/json) + response=$(curl -s https://pypi.org/pypi/apache-hamilton-ui/json) version=$(echo "$response" | jq -r '.info.version') echo "latest_version=$version" >> $GITHUB_ENV diff --git a/README.md b/README.md index e6283d7f0..1fd0edff4 100644 --- a/README.md +++ b/README.md @@ -24,14 +24,14 @@ under the License. Python supported - - PyPi Version + + PyPi Version - - Total Downloads + + Total Downloads - - Total Monthly Downloads + + Total Monthly Downloads
@@ -73,13 +73,13 @@ Apache Hamilton brings modularity and structure to any Python application moving Apache Hamilton supports Python 3.8+. We include the optional `visualization` dependency to display our Apache Hamilton DAG. For visualizations, [Graphviz](https://graphviz.org/download/) needs to be installed on your system separately. ```bash -pip install "sf-hamilton[visualization]" +pip install "apache-hamilton[visualization]" ``` To use the Apache Hamilton UI, install the `ui` and `sdk` dependencies. ```bash -pip install "sf-hamilton[ui,sdk]" +pip install "apache-hamilton[ui,sdk]" ``` To try Apache Hamilton in the browser, visit [www.tryhamilton.dev](https://www.tryhamilton.dev/?utm_source=README) diff --git a/contrib/README.md b/contrib/README.md index 9570ecc1b..f481388df 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -27,7 +27,7 @@ under the License. > > While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. -Here you'll find documentation about the sf-hamilton-contrib package that curates a collection of Apache Hamilton Dataflows that are +Here you'll find documentation about the apache-hamilton-contrib package that curates a collection of Apache Hamilton Dataflows that are ready to be used in your own projects. They are user-contributed and maintained, with the goal of making it easier for you to get started with Apache Hamilton. @@ -47,7 +47,7 @@ production purposes as you can version-lock your dependencies. To install the package, run: ```bash -pip install sf-hamilton-contrib --upgrade +pip install apache-hamilton-contrib --upgrade ``` Once installed, you can import the dataflows as follows. diff --git a/contrib/docs/compile_docs.py b/contrib/docs/compile_docs.py index f15270817..549d9b541 100644 --- a/contrib/docs/compile_docs.py +++ b/contrib/docs/compile_docs.py @@ -171,7 +171,7 @@ def dataflows_with_everything( ### Use published library version ```bash -pip install sf-hamilton-contrib --upgrade # make sure you have the latest +pip install apache-hamilton-contrib --upgrade # make sure you have the latest ``` import example2 from '!!raw-loader!./example2.py'; @@ -226,7 +226,7 @@ def dataflows_with_everything( ### Use published library version ```bash -pip install sf-hamilton-contrib --upgrade # make sure you have the latest +pip install apache-hamilton-contrib --upgrade # make sure you have the latest ``` import example2 from '!!raw-loader!./example2.py'; diff --git a/contrib/docs/docs/README.md b/contrib/docs/docs/README.md index 81cba280e..0c19d708d 100644 --- a/contrib/docs/docs/README.md +++ b/contrib/docs/docs/README.md @@ -47,7 +47,7 @@ There are two methods to get access to dataflows presented here. Assumptions: 1. You are familiar with Apache Hamilton and have it installed. If not, take -[15 minutes to learn Apache Hamilton in your browser](https://www.tryhamilton.dev/) and then `pip install sf-hamilton` to get started. +[15 minutes to learn Apache Hamilton in your browser](https://www.tryhamilton.dev/) and then `pip install apache-hamilton` to get started. Come back here when you're ready to use Apache Hamilton. 2. You have the requisite python dependencies installed on your system. You'll get import errors if you don't. Don't know what you need? Scroll to the bottom of a dataflow to find the requirements. We're working on convenience functions to help! @@ -85,7 +85,7 @@ production purposes as you can version-lock your dependencies. To install the package, run: ```bash -pip install sf-hamilton-contrib --upgrade +pip install apache-hamilton-contrib --upgrade ``` Once installed, you can import the dataflows as follows. diff --git a/contrib/docs/src/components/example_code.tsx b/contrib/docs/src/components/example_code.tsx index cf19afb00..270277916 100644 --- a/contrib/docs/src/components/example_code.tsx +++ b/contrib/docs/src/components/example_code.tsx @@ -30,7 +30,7 @@ dr = driver.Driver({}, text_summarization) `; export const example2 = `from hamilton import driver -# pip install sf-hamilton-contrib==0.0.1rc1 +# pip install apache-hamilton-contrib==0.0.1rc1 from hamilton.contrib.user.zilto import text_summarization dr = driver.Driver({}, text_summarization) # use the driver`; diff --git a/contrib/hamilton/contrib/dagworks/sphinx_doc_chunking/requirements.txt b/contrib/hamilton/contrib/dagworks/sphinx_doc_chunking/requirements.txt index 27dae66ce..9a06c390a 100644 --- a/contrib/hamilton/contrib/dagworks/sphinx_doc_chunking/requirements.txt +++ b/contrib/hamilton/contrib/dagworks/sphinx_doc_chunking/requirements.txt @@ -1,6 +1,6 @@ +apache-hamilton[dask] +# optionally install Ray, or Dask, or both +apache-hamilton[ray] +apache-hamilton[visualization] langchain langchain-core -sf-hamilton[dask] -# optionally install Ray, or Dask, or both -sf-hamilton[ray] -sf-hamilton[visualization] diff --git a/contrib/hamilton/contrib/user/skrawcz/air_quality_analysis/requirements.txt b/contrib/hamilton/contrib/user/skrawcz/air_quality_analysis/requirements.txt index a8b0c26b4..802477ed2 100644 --- a/contrib/hamilton/contrib/user/skrawcz/air_quality_analysis/requirements.txt +++ b/contrib/hamilton/contrib/user/skrawcz/air_quality_analysis/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton numpy scipy -sf-hamilton diff --git a/contrib/hamilton/contrib/user/skrawcz/customize_embeddings/requirements.txt b/contrib/hamilton/contrib/user/skrawcz/customize_embeddings/requirements.txt index b28292e07..bd1307caa 100644 --- a/contrib/hamilton/contrib/user/skrawcz/customize_embeddings/requirements.txt +++ b/contrib/hamilton/contrib/user/skrawcz/customize_embeddings/requirements.txt @@ -1,3 +1,4 @@ +apache-hamilton numpy openai pandas @@ -5,6 +6,5 @@ pandera plotly requests scikit-learn -sf-hamilton tenacity torch diff --git a/contrib/hamilton/contrib/user/skrawcz/fine_tuning/requirements.txt b/contrib/hamilton/contrib/user/skrawcz/fine_tuning/requirements.txt index f53a86aef..c52ab6691 100644 --- a/contrib/hamilton/contrib/user/skrawcz/fine_tuning/requirements.txt +++ b/contrib/hamilton/contrib/user/skrawcz/fine_tuning/requirements.txt @@ -1,4 +1,5 @@ absl-py +apache-hamilton datasets evaluate numpy @@ -6,7 +7,6 @@ pandas peft rouge-score scikit-learn -sf-hamilton torch tqdm transformers diff --git a/contrib/hamilton/contrib/user/zilto/lancedb_vdb/requirements.txt b/contrib/hamilton/contrib/user/zilto/lancedb_vdb/requirements.txt index 0ba826c35..fd6e2986e 100644 --- a/contrib/hamilton/contrib/user/zilto/lancedb_vdb/requirements.txt +++ b/contrib/hamilton/contrib/user/zilto/lancedb_vdb/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton[visualization] lancedb numpy pyarrow -sf-hamilton[visualization] diff --git a/contrib/hamilton/contrib/user/zilto/webscraper/requirements.txt b/contrib/hamilton/contrib/user/zilto/webscraper/requirements.txt index 355b16d8c..c009c9e02 100644 --- a/contrib/hamilton/contrib/user/zilto/webscraper/requirements.txt +++ b/contrib/hamilton/contrib/user/zilto/webscraper/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton[visualization] beautifulsoup4 lxml requests -sf-hamilton[visualization] diff --git a/contrib/hamilton/contrib/user/zilto/xgboost_optuna/requirements.txt b/contrib/hamilton/contrib/user/zilto/xgboost_optuna/requirements.txt index e3abdc007..63ead54a4 100644 --- a/contrib/hamilton/contrib/user/zilto/xgboost_optuna/requirements.txt +++ b/contrib/hamilton/contrib/user/zilto/xgboost_optuna/requirements.txt @@ -1,6 +1,6 @@ +apache-hamilton[visualization] numpy optuna pandas scikit-learn -sf-hamilton[visualization] xgboost diff --git a/contrib/requirements.txt b/contrib/requirements.txt index f9ba9ac8b..59045ac73 100644 --- a/contrib/requirements.txt +++ b/contrib/requirements.txt @@ -1 +1 @@ -sf-hamilton +apache-hamilton diff --git a/contrib/scripts/version_check.py b/contrib/scripts/version_check.py index 1b0c39fd5..834224eb6 100644 --- a/contrib/scripts/version_check.py +++ b/contrib/scripts/version_check.py @@ -28,7 +28,7 @@ CURRENT_VERSION = ".".join(map(str, VERSION)) # Replace 'your-package-name' with your actual package name on PyPI -response = requests.get("https://pypi.org/pypi/sf-hamilton-contrib/json") +response = requests.get("https://pypi.org/pypi/apache-hamilton-contrib/json") data = response.json() pypi_version = data["info"]["version"] diff --git a/dev_tools/vscode_extension/resources/walkthrough/3_pythonDependencies.md b/dev_tools/vscode_extension/resources/walkthrough/3_pythonDependencies.md index a1fc82ce8..8726221ae 100644 --- a/dev_tools/vscode_extension/resources/walkthrough/3_pythonDependencies.md +++ b/dev_tools/vscode_extension/resources/walkthrough/3_pythonDependencies.md @@ -19,6 +19,6 @@ under the License. # Install Python dependencies -Install the ``sf-hamilton`` and the visualization dependencies in your Python environment +Install the ``apache-hamilton`` and the visualization dependencies in your Python environment -``pip install sf-hamilton[lsp]`` +``pip install apache-hamilton[lsp]`` diff --git a/dev_tools/vscode_extension/src/extension.ts b/dev_tools/vscode_extension/src/extension.ts index 953e23d67..4fdf27fc8 100644 --- a/dev_tools/vscode_extension/src/extension.ts +++ b/dev_tools/vscode_extension/src/extension.ts @@ -32,12 +32,12 @@ const checkPythonDependencies = (pythonPath: string) => { } function installDependenciesMessage(pythonPath: string) { - let installLSP = "Install sf-hamilton[lsp]" + let installLSP = "Install apache-hamilton[lsp]" vscode.window.showInformationMessage("Missing the Hamilton language server.", installLSP) .then(selection => { if (selection === installLSP) { - execSync(`${pythonPath} -m pip install 'sf-hamilton-lsp'`) + execSync(`${pythonPath} -m pip install 'apache-hamilton-lsp'`) vscode.commands.executeCommand("workbench.action.reloadWindow") } }) diff --git a/docs/concepts/best-practices/using-within-your-etl-system.rst b/docs/concepts/best-practices/using-within-your-etl-system.rst index b41634a97..743644037 100644 --- a/docs/concepts/best-practices/using-within-your-etl-system.rst +++ b/docs/concepts/best-practices/using-within-your-etl-system.rst @@ -36,6 +36,6 @@ ETL Recipe #. Write Hamilton functions & `“driver”` code. #. Publish your Hamilton functions in a package, or import via other means (e.g. checkout a repository & include in python path). -#. Include `sf-hamilton` as a python dependency +#. Include `apache-hamilton` as a python dependency #. Have your ETL system execute your “driver” code. #. Profit. diff --git a/docs/concepts/function-modifiers.rst b/docs/concepts/function-modifiers.rst index 7e63c297c..b5ea15359 100644 --- a/docs/concepts/function-modifiers.rst +++ b/docs/concepts/function-modifiers.rst @@ -174,13 +174,13 @@ The next snippet checks if the returned Series is of type ``np.int32``, which is pandera support ~~~~~~~~~~~~~~~ -Apache Hamilton has a pandera plugin for data validation that you can install with ``pip install sf-hamilton[pandera]``. Then, you can pass a pandera schema (for DataFrame or Series) to ``@check_output(schema=...)``. +Apache Hamilton has a pandera plugin for data validation that you can install with ``pip install apache-hamilton[pandera]``. Then, you can pass a pandera schema (for DataFrame or Series) to ``@check_output(schema=...)``. pydantic support ~~~~~~~~~~~~~~~~ -Apache Hamilton also supports data validation of pydantic models, which can be enabled with ``pip install sf-hamilton[pydantic]``. With pydantic installed, you can pass any subclass of the pydantic base model to ``@check_output(model=...)``. Pydantic validation is performed in strict mode, meaning that raw values will not be coerced to the model's types. For more information on strict mode see the `pydantic docs `_. +Apache Hamilton also supports data validation of pydantic models, which can be enabled with ``pip install apache-hamilton[pydantic]``. With pydantic installed, you can pass any subclass of the pydantic base model to ``@check_output(model=...)``. Pydantic validation is performed in strict mode, meaning that raw values will not be coerced to the model's types. For more information on strict mode see the `pydantic docs `_. Split node output into *n* nodes diff --git a/docs/conf.py b/docs/conf.py index 446e09bb7..b473238cd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -113,7 +113,7 @@ # check if the current commit is tagged as a release (vX.Y.Z) and set the version GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"]) current_tag = GIT_TAG_OUTPUT.decode().strip() -if re.match(r"^sf-hamilton-(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$", current_tag): +if re.match(r"^apache-hamilton-(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$", current_tag): version = current_tag else: version = "latest" diff --git a/docs/get-started/install.md b/docs/get-started/install.md index 04055a89d..04afcabcc 100644 --- a/docs/get-started/install.md +++ b/docs/get-started/install.md @@ -31,13 +31,13 @@ For help with python/pip/managing virtual environments see the [python docs](htt ## Installing with pip -Apache Hamilton is published on [pypi](https://pypi.org/project/sf-hamilton/) under ``sf-hamilton``. To install, run: +Apache Hamilton is published on [pypi](https://pypi.org/project/apache-hamilton/) under ``apache-hamilton``. To install, run: -`pip install sf-hamilton` +`pip install apache-hamilton` To use the DAG visualization functionality, instead install with -`pip install sf-hamilton[visualization]` +`pip install apache-hamilton[visualization]` *Note: for visualization you may additionally need to install graphviz externally -- see* [graphviz](https://graphviz.org/download/) *for instructions on the correct way for your @@ -47,7 +47,7 @@ operating system.* Apache Hamilton is also available on conda if you prefer: -`conda install -c hamilton-opensource sf-hamilton` +`conda install -c hamilton-opensource apache-hamilton` ## Installing from source diff --git a/docs/get-started/your-first-dataflow.rst b/docs/get-started/your-first-dataflow.rst index 9e8799a12..9ccf3859b 100644 --- a/docs/get-started/your-first-dataflow.rst +++ b/docs/get-started/your-first-dataflow.rst @@ -105,7 +105,7 @@ To actually run the dataflow, we'll need to write :doc:`a driver <../concepts/dr ] # let's create the dataframe! df = dr.execute(output_columns, inputs=initial_columns) - # `pip install sf-hamilton[visualization]` earlier you can also do + # `pip install apache-hamilton[visualization]` earlier you can also do # dr.visualize_execution(output_columns,'./my_dag.png', {}) print(df) diff --git a/docs/hamilton-ui/ui.rst b/docs/hamilton-ui/ui.rst index eeec6b2de..3e1811eba 100644 --- a/docs/hamilton-ui/ui.rst +++ b/docs/hamilton-ui/ui.rst @@ -29,7 +29,7 @@ To run the hamilton UI in local mode, you can do the following: .. code-block:: bash - pip install "sf-hamilton[ui,sdk]" + pip install "apache-hamilton[ui,sdk]" hamilton ui # python -m hamilton.cli.__main__ ui # on windows @@ -148,7 +148,7 @@ First, install the SDK: .. code-block:: bash - pip install "sf-hamilton[sdk]" + pip install "apache-hamilton[sdk]" Then, navigate to the project page (dashboard/projects), in the running UI, and click the green ``+ New DAG`` button. diff --git a/docs/hamilton-vscode/language_server.rst b/docs/hamilton-vscode/language_server.rst index f04c0560c..ac85718d4 100644 --- a/docs/hamilton-vscode/language_server.rst +++ b/docs/hamilton-vscode/language_server.rst @@ -20,12 +20,12 @@ You can also manually install the language server in your Python environment via .. code:: shell - pip install "sf-hamilton[lsp]" + pip install "apache-hamilton[lsp]" Developers ---------- -If you want to dig in the internals of the language server and integrate it with another IDE, you can find the source code in the ``dev tools/`` section of the `Apache Hamilton GitHub repository `_. It is also directly available on PyPi at `sf-hamilton-lsp `_. +If you want to dig in the internals of the language server and integrate it with another IDE, you can find the source code in the ``dev tools/`` section of the `Apache Hamilton GitHub repository `_. It is also directly available on PyPi at `apache-hamilton-lsp `_. Note that the package name is ``hamilton_lsp`` when used directly via Python code. diff --git a/docs/how-tos/cli-reference.md b/docs/how-tos/cli-reference.md index 7350b4551..13bffcb6a 100644 --- a/docs/how-tos/cli-reference.md +++ b/docs/how-tos/cli-reference.md @@ -25,7 +25,7 @@ This page covers the Apache Hamilton CLI. It is built directly from the CLI, but The dependencies for the Apache Hamilton CLI can be installed via ```console -pip install sf-hamilton[cli] +pip install apache-hamilton[cli] ``` You can verify the installation with diff --git a/docs/how-tos/pre-commit-hooks.md b/docs/how-tos/pre-commit-hooks.md index 2f61a1b6c..dc36ab269 100644 --- a/docs/how-tos/pre-commit-hooks.md +++ b/docs/how-tos/pre-commit-hooks.md @@ -73,7 +73,7 @@ The preferred way of working with pre-commit hooks is through the [prek](https:/ pre-commit hooks are great developer tools, but off-the-shelf solutions aren't aware of the Apache Hamilton framework. Hence, we developed a pre-commit hook to help you author Apache Hamilton dataflows! Under the hood, they leverage the `hamilton` CLI, so if you are unfamiliar with it, feel free to install it and view the `--help` messages. ```console -pip install sf-hamilton[cli] +pip install apache-hamilton[cli] hamilton --help ``` diff --git a/docs/how-tos/use-in-jupyter-notebook.md b/docs/how-tos/use-in-jupyter-notebook.md index 9f242c496..5b48f39a6 100644 --- a/docs/how-tos/use-in-jupyter-notebook.md +++ b/docs/how-tos/use-in-jupyter-notebook.md @@ -136,7 +136,7 @@ I assume you already have this step set up. But just in case you don’t: ```bash pip install jupyterlab -pip install sf-hamilton +pip install apache-hamilton ``` Then to start the notebook server it should just be: diff --git a/docs/main.md b/docs/main.md index 324bc10fc..a7a764e95 100644 --- a/docs/main.md +++ b/docs/main.md @@ -21,8 +21,8 @@ under the License.
Apache Hamilton Slack Twitter - Total Downloads - Total Monthly Downloads + Total Downloads + Total Monthly Downloads
Apache Hamilton (incubating) is a general-purpose framework to write dataflows using regular Python functions. At the core, each function defines a transformation and its parameters indicates its dependencies. Apache Hamilton automatically connects individual functions into a [Directed Acyclic Graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) that can be executed, visualized, optimized, and reported on. diff --git a/docs/reference/decorators/with_columns.rst b/docs/reference/decorators/with_columns.rst index fd9918714..a5a11fb03 100644 --- a/docs/reference/decorators/with_columns.rst +++ b/docs/reference/decorators/with_columns.rst @@ -36,7 +36,7 @@ PySpark This is part of the hamilton pyspark integration. To install, run: -`pip install sf-hamilton[pyspark]` +`pip install apache-hamilton[pyspark]` **Reference Documentation** diff --git a/docs/reference/lifecycle-hooks/Narwhals.rst b/docs/reference/lifecycle-hooks/Narwhals.rst index 54e1a7a37..4df7ca27c 100644 --- a/docs/reference/lifecycle-hooks/Narwhals.rst +++ b/docs/reference/lifecycle-hooks/Narwhals.rst @@ -5,7 +5,7 @@ plugins.h_narhwals.NarwhalsAdapter Provides a convenience wrapper for the `Narwhals` library; use the Narwhals decorator underneath. Must have `Narwhals` installed to use it: -`pip install "sf-hamilton[narwhals]"` +`pip install "apache-hamilton[narwhals]"` .. autoclass:: hamilton.plugins.h_narwhals.NarwhalsAdapter @@ -20,7 +20,7 @@ plugins.h_narhwals.NarwhalsDataFrameResultBuilder Result builder to be used with the NarwhalsAdapter. Must have `Narwhals` installed to use it: -`pip install "sf-hamilton[narwhals]"` +`pip install "apache-hamilton[narwhals]"` .. autoclass:: hamilton.plugins.h_narwhals.NarwhalsDataFrameResultBuilder diff --git a/docs/reference/lifecycle-hooks/ProgressBar.rst b/docs/reference/lifecycle-hooks/ProgressBar.rst index 9e58d56e9..3c85ce892 100644 --- a/docs/reference/lifecycle-hooks/ProgressBar.rst +++ b/docs/reference/lifecycle-hooks/ProgressBar.rst @@ -4,7 +4,7 @@ plugins.h_tqdm.ProgressBar Provides a progress bar for Apache Hamilton execution. Must have `tqdm` installed to use it: -`pip install sf-hamilton[tqdm]` (use quotes if using zsh) +`pip install apache-hamilton[tqdm]` (use quotes if using zsh) .. autoclass:: hamilton.plugins.h_tqdm.ProgressBar diff --git a/docs/reference/lifecycle-hooks/RichProgressBar.rst b/docs/reference/lifecycle-hooks/RichProgressBar.rst index 9390e547c..ca9d738de 100644 --- a/docs/reference/lifecycle-hooks/RichProgressBar.rst +++ b/docs/reference/lifecycle-hooks/RichProgressBar.rst @@ -4,7 +4,7 @@ plugins.h_rich.RichProgressBar Provides a progress bar for Apache Hamilton execution. Must have `rich` installed to use it: -`pip install sf-hamilton[rich]` (use quotes if using zsh) +`pip install apache-hamilton[rich]` (use quotes if using zsh) .. autoclass:: hamilton.plugins.h_rich.RichProgressBar diff --git a/docs/reference/lifecycle-hooks/SlackNotifierHook.rst b/docs/reference/lifecycle-hooks/SlackNotifierHook.rst index dfcbbd5fe..22856ba32 100644 --- a/docs/reference/lifecycle-hooks/SlackNotifierHook.rst +++ b/docs/reference/lifecycle-hooks/SlackNotifierHook.rst @@ -4,7 +4,7 @@ plugins.h_slack.SlackNotifier Provides a Slack notifier for Apache Hamilton execution. Must have `slack_sdk` installed to use it: -`pip install sf-hamilton[slack]` (use quotes if using zsh) +`pip install apache-hamilton[slack]` (use quotes if using zsh) .. autoclass:: hamilton.plugins.h_slack.SlackNotifier diff --git a/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb b/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb index ea229c8fe..6c500d3c6 100644 --- a/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb +++ b/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[sdk,visualization] falkordb openai" + "%pip install apache-hamilton[sdk,visualization] falkordb openai" ] }, { diff --git a/examples/LLM_Workflows/GraphRAG/requirements.txt b/examples/LLM_Workflows/GraphRAG/requirements.txt index c76190ecd..76cbdd939 100644 --- a/examples/LLM_Workflows/GraphRAG/requirements.txt +++ b/examples/LLM_Workflows/GraphRAG/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton[sdk,visualization] burr[start,graphviz] falkordb==1.0.4 openai -sf-hamilton[sdk,visualization] diff --git a/examples/LLM_Workflows/NER_Example/notebook.ipynb b/examples/LLM_Workflows/NER_Example/notebook.ipynb index 2e5155f39..0225ead11 100644 --- a/examples/LLM_Workflows/NER_Example/notebook.ipynb +++ b/examples/LLM_Workflows/NER_Example/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] sentence_transformers datasets lancedb -qU" + "%pip install apache-hamilton[visualization] sentence_transformers datasets lancedb -qU" ] }, { diff --git a/examples/LLM_Workflows/NER_Example/requirements.txt b/examples/LLM_Workflows/NER_Example/requirements.txt index 7b6a55b65..42a8e1e8e 100644 --- a/examples/LLM_Workflows/NER_Example/requirements.txt +++ b/examples/LLM_Workflows/NER_Example/requirements.txt @@ -1,7 +1,7 @@ +apache-hamilton[visualization, sdk] datasets lancedb numpy sentence_transformers -sf-hamilton[visualization, sdk] torch tqdm diff --git a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/requirements.txt b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/requirements.txt index e9a45356a..33ab380e8 100644 --- a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/requirements.txt +++ b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/requirements.txt @@ -1,6 +1,6 @@ +apache-hamilton[visualization] langchain langchain-core openai pandas requests -sf-hamilton[visualization] diff --git a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb index 1350c4c1b..2011e8ed5 100644 --- a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb +++ b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -55,7 +55,7 @@ "outputs": [], "source": [ "# install the required packages to run this notebook\n", - "!pip install sf-hamilton[visualization] openai langchain langchain-core requests pandas" + "!pip install apache-hamilton[visualization] openai langchain langchain-core requests pandas" ] }, { diff --git a/examples/LLM_Workflows/image_telephone/notebook.ipynb b/examples/LLM_Workflows/image_telephone/notebook.ipynb index fe6454fb8..9453a6b46 100644 --- a/examples/LLM_Workflows/image_telephone/notebook.ipynb +++ b/examples/LLM_Workflows/image_telephone/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] burr openai pillow" + "%pip install apache-hamilton[visualization] burr openai pillow" ] }, { diff --git a/examples/LLM_Workflows/image_telephone/requirements.txt b/examples/LLM_Workflows/image_telephone/requirements.txt index 82c5a7de5..afd975fde 100644 --- a/examples/LLM_Workflows/image_telephone/requirements.txt +++ b/examples/LLM_Workflows/image_telephone/requirements.txt @@ -1,4 +1,6 @@ altair +apache-hamilton +apache-hamilton-contrib boto3 burr openai @@ -7,8 +9,6 @@ pandas pillow s3fs scikit-learn -sf-hamilton -sf-hamilton-contrib st-files-connection streamlit streamlit-super-slider diff --git a/examples/LLM_Workflows/knowledge_retrieval/requirements.txt b/examples/LLM_Workflows/knowledge_retrieval/requirements.txt index 5965215a4..ace4cee77 100644 --- a/examples/LLM_Workflows/knowledge_retrieval/requirements.txt +++ b/examples/LLM_Workflows/knowledge_retrieval/requirements.txt @@ -1,10 +1,10 @@ +apache-hamilton[visualization] arxiv openai==0.28.0 pandas PyPDF2 requests scipy -sf-hamilton[visualization] tenacity termcolor tiktoken==0.3.3 diff --git a/examples/LLM_Workflows/langchain_comparison/requirements.txt b/examples/LLM_Workflows/langchain_comparison/requirements.txt index fb6a3c402..7ac67b8ff 100644 --- a/examples/LLM_Workflows/langchain_comparison/requirements.txt +++ b/examples/LLM_Workflows/langchain_comparison/requirements.txt @@ -1,5 +1,5 @@ anthropic +apache-hamilton[visualization] lanchain_openai langchain openai -sf-hamilton[visualization] diff --git a/examples/LLM_Workflows/llm_logging/requirements.txt b/examples/LLM_Workflows/llm_logging/requirements.txt index f1fa82ef0..baa4c4104 100644 --- a/examples/LLM_Workflows/llm_logging/requirements.txt +++ b/examples/LLM_Workflows/llm_logging/requirements.txt @@ -1,7 +1,7 @@ +apache-hamilton[visualization] fastapi fastui openai pypdf -sf-hamilton[visualization] tiktoken diff --git a/examples/LLM_Workflows/modular_llm_stack/requirements.txt b/examples/LLM_Workflows/modular_llm_stack/requirements.txt index 2aef03be7..4c13f7b98 100644 --- a/examples/LLM_Workflows/modular_llm_stack/requirements.txt +++ b/examples/LLM_Workflows/modular_llm_stack/requirements.txt @@ -1,3 +1,5 @@ +apache-hamilton +apache-hamilton[visualization] # comment out the vector DB packages you don't need click cohere @@ -11,6 +13,4 @@ pandas pinecone-client qdrant-client sentence-transformers -sf-hamilton -sf-hamilton[visualization] weaviate-client diff --git a/examples/LLM_Workflows/neo4j_graph_rag/requirements.txt b/examples/LLM_Workflows/neo4j_graph_rag/requirements.txt index a2cfc9387..04e9226ee 100644 --- a/examples/LLM_Workflows/neo4j_graph_rag/requirements.txt +++ b/examples/LLM_Workflows/neo4j_graph_rag/requirements.txt @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. +apache-hamilton>=1.73.0 neo4j>=5.18.0 openai>=1.30.0 pandas>=2.0.0 python-dotenv>=1.0.0 -sf-hamilton>=1.73.0 tqdm>=4.66.0 diff --git a/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb b/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb index 4127ced90..5b384bb5b 100644 --- a/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb +++ b/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/LLM_Workflows/observability_openllmetry/requirements.txt b/examples/LLM_Workflows/observability_openllmetry/requirements.txt index d83512fa7..95dc786e2 100644 --- a/examples/LLM_Workflows/observability_openllmetry/requirements.txt +++ b/examples/LLM_Workflows/observability_openllmetry/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton openai opentelemetry-instrumentation-openai -sf-hamilton traceloop-sdk diff --git a/examples/LLM_Workflows/pdf_summarizer/backend/requirements.txt b/examples/LLM_Workflows/pdf_summarizer/backend/requirements.txt index 633eeeee5..8b7342553 100644 --- a/examples/LLM_Workflows/pdf_summarizer/backend/requirements.txt +++ b/examples/LLM_Workflows/pdf_summarizer/backend/requirements.txt @@ -1,10 +1,10 @@ +apache-hamilton[visualization] dagworks-sdk>=0.3.2 # DAGWorks Tracker fastapi openai pydantic PyPDF2 python-multipart -sf-hamilton[visualization] tenacity tiktoken tqdm diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/requirements.txt b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/requirements.txt index 7edb5f09b..7b564625b 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/requirements.txt +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/requirements.txt @@ -1,7 +1,7 @@ +apache-hamilton[visualization] openai PyPDF2 pyspark -sf-hamilton[visualization] tenacity tiktoken tqdm diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb index 44e3b0824..82869970e 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py b/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py index 4b7f75464..045771e38 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py @@ -69,7 +69,7 @@ def local_pdfs( """List of local PDF files, either string paths or in-memory files (on the FastAPI server) NOTE. This function is overriden by the driver to use arbitrary local PDFs and don't need to query arxiv. It is necessary because Parallelizable and Collect nodes - cannot be overriden safely at the moment (sf-hamilton==1.26.0) + cannot be overriden safely at the moment (apache-hamilton==1.26.0) """ return arxiv_pdf_path_collection diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/backend/requirements.txt b/examples/LLM_Workflows/retrieval_augmented_generation/backend/requirements.txt index 875228c6c..fe17b4d8d 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/backend/requirements.txt +++ b/examples/LLM_Workflows/retrieval_augmented_generation/backend/requirements.txt @@ -1,3 +1,4 @@ +apache-hamilton[visualization] == 1.27.3 arxiv fastapi openai @@ -5,7 +6,6 @@ pandas pydantic PyPDF2 python-multipart -sf-hamilton[visualization] == 1.27.3 tenacity tiktoken uvicorn diff --git a/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb b/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb index 5db8d4570..c9eba03c9 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb +++ b/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -5502,7 +5502,7 @@ " '
  • Publish your Hamilton functions in a package, or '\n", " 'import via other means (e.g. checkout a repository & '\n", " 'include in python path).

  • \\n'\n", - " '
  • Include sf-hamilton as a python '\n", + " '

  • Include apache-hamilton as a python '\n", " 'dependency

  • \\n'\n", " '
  • Have your ETL system execute your “driver” '\n", " 'code.

  • \\n'\n", @@ -5515,7 +5515,7 @@ " Document(page_content='Dagster \\n✅ \\nPrefect \\n✅ (see [prefect example](https://github.com/apache/hamilton/tree/main/examples/prefect)) \\nKubeflow Pipelines \\n✅ \\nCRON \\n✅ \\ndbt \\n✅ (see dbt example) \\nkubernetes', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", " Document(page_content='kubernetes \\n✅ but you need to setup kubernetes to run an image that can run python code - e.g. see Running a python application on kubernetes \\ndocker \\n✅ but you need to setup a docker image that can execute python code.', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", " Document(page_content='… in general if it runs python 3.7+ … \\n✅', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", - " Document(page_content='Write Hamilton functions & “driver” code. \\nPublish your Hamilton functions in a package, or import via other means (e.g. checkout a repository & include in python path). \\nInclude sf-hamilton as a python dependency', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'}),\n", + " Document(page_content='Write Hamilton functions & “driver” code. \\nPublish your Hamilton functions in a package, or import via other means (e.g. checkout a repository & include in python path). \\nInclude apache-hamilton as a python dependency', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'}),\n", " Document(page_content='Have your ETL system execute your “driver” code. \\nProfit.', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'})],\n", " 'url': 'https://hamilton.dagworks.io/en/latest/concepts/best-practices/using-within-your-etl-system/'}\n", "{'article_text': '
    \\n'\n", @@ -6776,7 +6776,7 @@ " 'you can install with pip install sf-hamilton[pandera]. Then, you '\n", + " 'class=\"pre\">apache-hamilton[pandera]. Then, you '\n", " 'can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).

    \\n'\n", @@ -7591,7 +7591,7 @@ " Document(page_content='(instead of functions). See hamilton/data_quality/default_validators.py or reach out on Slack for help!', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", " Document(page_content='Note \\nAs you see, validation steps effectively add nodes to the dataflow and the visualization. This helps trace which specific check failed for instance, but it can make visualizations harder to read.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", " Document(page_content='You can hide these nodes using the custom visualization style feature (see Apply custom style) by applying the style {\"style\": \"invis\"} to nodes with the tag hamilton.data_quality.source_node. This will only keep the original nodes and their _raw variant.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", - " Document(page_content='Hamilton has a pandera plugin for data validation that you can install with pip install sf-hamilton[pandera]. Then, you can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': 'pandera support¶'}),\n", + " Document(page_content='Hamilton has a pandera plugin for data validation that you can install with pip install apache-hamilton[pandera]. Then, you can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': 'pandera support¶'}),\n", " Document(page_content='Sometimes, your node outputs multiple values that you would like to name and make available to other nodes. These function modifiers act on the node output / function return. \\nNote', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶'}),\n", " Document(page_content='Note \\nTo add metadata to extracted nodes, use @tag_output, which works just like @tag.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶'}),\n", " Document(page_content='A good example is splitting a dataset into train, validation, and test splits. We will use @extract_fields, which requires specifying in a dictionary the field_name: field_type of each field.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶', 'Header 3': '@extract_fields¶'}),\n", @@ -10207,7 +10207,7 @@ " '
    pip '\n",
           "                 'install "sf-hamilton[sdk]"\\n'\n",
    +      "                 'class=\"s2\">"apache-hamilton[sdk]"\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", " '

    Then, navigate to the project page (dashboard/projects), '\n", @@ -10384,7 +10384,7 @@ " Document(page_content='Start the frontend server \\nThis takes a bit of time! So be patient. The server will be running on port 8242. Then navigate to http://localhost:8242 in your browser, and enter your email (this will be the username used within the app).', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", " Document(page_content='If building the Docker containers from scratch, increase your Docker memory to 10gb or more – you can do this in the Docker Desktop settings. \\nTo build the images locally, you can run the following command:', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", " Document(page_content='# from the hamilton/ui directory ./deployment/dev.sh --build \\nThis will build the containers from scratch. If you just want to mount the local code, you can run just \\n./deployment/dev.sh', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", - " Document(page_content='Now that you have your server running, you can run a simple dataflow and watch it in the UI! You can follow instructions in the UI when you create a new project, or follow the instructions here. \\nFirst, install the SDK: \\npip install \"sf-hamilton[sdk]\"', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", + " Document(page_content='Now that you have your server running, you can run a simple dataflow and watch it in the UI! You can follow instructions in the UI when you create a new project, or follow the instructions here. \\nFirst, install the SDK: \\npip install \"apache-hamilton[sdk]\"', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='Then, navigate to the project page (dashboard/projects), in the running UI, and click the green + New DAG button. \\nRemember the project ID – you’ll use it for the next steps. \\nAdd the following adapter to your code if you have existing Hamilton code:', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='from hamilton_sdk import adapters tracker = adapters.HamiltonTracker( project_id=PROJECT_ID_FROM_ABOVE, username=\"EMAIL_YOU_PUT_IN_THE_UI\", dag_name=\"my_version_of_the_dag\", tags={\"environment\": \"DEV\", \"team\": \"MY_TEAM\", \"version\": \"X\"} ) dr = (', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='\"version\": \"X\"} ) dr = ( driver.Builder() .with_config(your_config) .with_modules(*your_modules) .with_adapters(tracker) .build() )', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", @@ -11071,18 +11071,18 @@ " 'href=\"#installing-with-pip\" title=\"Link to this '\n", " 'heading\">¶\\n'\n", " '

    Hamilton is published on pypi under '\n", + " 'href=\"https://pypi.org/project/apache-hamilton/\">pypi under '\n", " 'sf-hamilton. To install, run:

    \\n'\n", + " 'class=\"pre\">apache-hamilton. To install, run:

    \\n'\n", " '

    pip install '\n", - " 'sf-hamilton

    \\n'\n", + " 'apache-hamilton

    \\n'\n", " '

    To use the DAG visualization functionality, instead '\n", " 'install with

    \\n'\n", " '

    pip install '\n", " 'sf-hamilton[visualization]

    \\n'\n", + " 'class=\"pre\">apache-hamilton[visualization]

    \\n'\n", " '

    Note: for visualization you may additionally need to '\n", " 'install graphviz externally – see\\n'\n", " 'conda install '\n", " '-c hamilton-opensource sf-hamilton

    \\n'\n", + " 'class=\"pre\">apache-hamilton

    \\n'\n", " '
    \\n'\n", " '
    \\n'\n", " '

    Installing from source',\n", " 'chunks': [Document(page_content='Installing hamilton is easy!'),\n", " Document(page_content='Hamilton is a lightweight framework with a variety of extensions/plugins. To get started, you’ll need the following: \\npython >= 3.8 \\npip \\nFor help with python/pip/managing virtual environments see the python docs.', metadata={'Header 1': 'Install¶'}),\n", - " Document(page_content='Hamilton is published on pypi under sf-hamilton. To install, run: \\npip install sf-hamilton \\nTo use the DAG visualization functionality, instead install with \\npip install sf-hamilton[visualization]', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", + " Document(page_content='Hamilton is published on pypi under apache-hamilton. To install, run: \\npip install apache-hamilton \\nTo use the DAG visualization functionality, instead install with \\npip install apache-hamilton[visualization]', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", " Document(page_content='Note: for visualization you may additionally need to install graphviz externally – see graphviz for instructions on the correct way for your operating system.', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", - " Document(page_content='Hamilton is also available on conda if you prefer: \\nconda install -c hamilton-opensource sf-hamilton', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with conda¶'}),\n", + " Document(page_content='Hamilton is also available on conda if you prefer: \\nconda install -c hamilton-opensource apache-hamilton', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with conda¶'}),\n", " Document(page_content='You can also download the code and run it from the source. \\ngit clone https://github.com/apache/hamilton.git cd hamilton pip install -e .', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing from source¶'})],\n", " 'url': 'https://hamilton.dagworks.io/en/latest/get-started/install/'}\n", "{'article_text': '
    \\n'\n", @@ -12229,7 +12229,7 @@ " 'inputs=initial_columns)\\n'\n", " ' # `pip install '\n", - " 'sf-hamilton[visualization]` earlier you can also do\\n'\n", + " 'apache-hamilton[visualization]` earlier you can also do\\n'\n", " ' # '\n", " 'dr.visualize_execution(output_columns,'./my_dag.dot', '\n", " '{})\\n'\n", @@ -12310,7 +12310,7 @@ " Document(page_content='# Instantiate a common spine for your pipeline index = pd.date_range(\"2022-01-01\", periods=6, freq=\"w\") initial_columns = { # load from actuals or wherever -- this is our initial data we use as input. # Note: these do not have to be all series, they could', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"to be all series, they could be scalar inputs. 'signups': pd.Series([1, 10, 50, 100, 200, 400], index=index), 'spend': pd.Series([10, 10, 20, 40, 40, 50], index=index), } # we need to tell hamilton where to load function definitions from config = {} # we\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"from config = {} # we don't have any configuration or invariant data for this example. dr = driver.Driver(config, my_functions) # can pass in multiple modules # we need to specify what we want in the final dataframe. output_columns = [ 'spend', 'signups',\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", - " Document(page_content=\"= [ 'spend', 'signups', 'avg_3wk_spend', 'acquisition_cost', ] # let's create the dataframe! df = dr.execute(output_columns, inputs=initial_columns) # `pip install sf-hamilton[visualization]` earlier you can also do #\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", + " Document(page_content=\"= [ 'spend', 'signups', 'avg_3wk_spend', 'acquisition_cost', ] # let's create the dataframe! df = dr.execute(output_columns, inputs=initial_columns) # `pip install apache-hamilton[visualization]` earlier you can also do #\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"earlier you can also do # dr.visualize_execution(output_columns,'./my_dag.dot', {}) print(df)\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content='Run the script with the following command: \\npython my_script.py \\nAnd you should see the following output:', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content='spend signups avg_3wk_spend acquisition_cost 2022-01-02 10 1 NaN 10.000 2022-01-09 10 10 NaN 1.000 2022-01-16 20 50 13.333333 0.400 2022-01-23 40 100 23.333333 0.400 2022-01-30 40 200 33.333333 0.200 2022-02-06 50 400 43.333333 0.125', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", @@ -12355,7 +12355,7 @@ " 'via

    \\n'\n", " '
    pip '\n",
    -      "                 'install sf-hamilton[cli]\\n'\n",
    +      "                 'install apache-hamilton[cli]\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", " '

    You can verify the installation with

    \\n'\n", @@ -12534,7 +12534,7 @@ " '
    \\n'\n", " '

    ',\n", " 'chunks': [Document(page_content='This page covers the Hamilton CLI. It is built directly from the CLI, but note that the command hamilton --help always provide the most accurate documentation.', metadata={'Header 1': 'Command line interface¶'}),\n", - " Document(page_content='The dependencies for the Hamilton CLI can be installed via \\npip install sf-hamilton[cli] \\nYou can verify the installation with \\nhamilton --help', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'Installation¶'}),\n", + " Document(page_content='The dependencies for the Hamilton CLI can be installed via \\npip install apache-hamilton[cli] \\nYou can verify the installation with \\nhamilton --help', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'Installation¶'}),\n", " Document(page_content='Options: \\n--verbose / --no-verbose: [default: no-verbose] \\n--json-out / --no-json-out: [default: no-json-out] \\n--install-completion: Install completion for the current shell.', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", " Document(page_content='--show-completion: Show completion for the current shell, to copy it or customize the installation. \\n--help: Show this message and exit. \\nCommands: \\nbuild: Build a single Driver with MODULES \\ndiff: Diff between the current MODULES and their…', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", " Document(page_content='version: Version NODES and DATAFLOW from dataflow… \\nview: Build and visualize dataflow with MODULES', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", @@ -13070,7 +13070,7 @@ " 'class=\"pre\">--help messages.

    \\n'\n", " '
    pip '\n",
    -      "                 'install sf-hamilton[cli]\\n'\n",
    +      "                 'install apache-hamilton[cli]\\n'\n",
           "                 'hamilton --help\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", @@ -13338,7 +13338,7 @@ " Document(page_content='install the hooks defined in .pre-commit-config.yaml \\npre-commit install \\nNow, hooks will automatically run on git commit \\nto manually run hooks \\npre-commit run --all-files', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Add pre-commit hooks to your project¶', 'Header 3': 'Steps to get started¶'}),\n", " Document(page_content='pre-commit hooks are great developer tools, but off-the-shelf solutions aren’t aware of the Hamilton framework. Hence, we developed a pre-commit hook to help you author Hamilton dataflows! Under the hood, they leverage the hamilton CLI, so if you are', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", " Document(page_content='the hamilton CLI, so if you are unfamiliar with it, feel free to install it and view the --help messages.', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", - " Document(page_content='pip install sf-hamilton[cli] hamilton --help', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", + " Document(page_content='pip install apache-hamilton[cli] hamilton --help', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", " Document(page_content='Hamilton doesn’t have many syntactic constraints, but there’s a few things we want to catch: \\nfunctions parameters and return are type annotated \\na node consistently has the same type (e.g., a parameter in multiple functions)', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", " Document(page_content='functions with a name starting with underscore (_) are ignored from the dataflow \\nfunctions with a @config decorator received a trailing double underscore with a suffix (e.g., hello__weekday(), hello__weekend())', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", " Document(page_content='Instead of reimplementing this logic, we can try to build the Hamilton Driver with the command hamilton build MODULES and catch errors. This also ensures the verification is always in sync with the actual build mechanism. This hook will help prevent us', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", @@ -14608,7 +14608,7 @@ " 'class=\"highlight\">
    pip '\n",
           "                 'install jupyterlab\\n'\n",
           "                 'pip install '\n",
    -      "                 'sf-hamilton\\n'\n",
    +      "                 'apache-hamilton\\n'\n",
           "                 '
    \\n'\n", " '\\n'\n", " '

    Then to start the notebook server it should just be:

    \\n'\n", @@ -14932,7 +14932,7 @@ " Document(page_content=\"# Step 3 - add the module to the driver and continue as usual dr = driver.Driver(config, some_functions, temp_module) df = dr.execute(['avg_3wk_spend', 'log_avg_3wk_spend'], inputs=input_data)\", metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '1 - Dynamically create modules within your notebook¶', 'Header 3': 'Using ad_hoc_utils to create a temporary module (e.g. use in google colab)¶'}),\n", " Document(page_content='Using a “temporary python module” will not enable scaling of computation by using Ray, Dask, or Pandas on Spark. So we suggest only using this approach for development purposes only.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '1 - Dynamically create modules within your notebook¶', 'Header 3': 'Caveat with this approach:¶'}),\n", " Document(page_content='This tutorial can also be found published on TDS.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶'}),\n", - " Document(page_content='I assume you already have this step set up. But just in case you don’t: \\npip install jupyterlab pip install sf-hamilton \\nThen to start the notebook server it should just be:', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 1 — Install Jupyter & Hamilton ¶'}),\n", + " Document(page_content='I assume you already have this step set up. But just in case you don’t: \\npip install jupyterlab pip install apache-hamilton \\nThen to start the notebook server it should just be:', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 1 — Install Jupyter & Hamilton ¶'}),\n", " Document(page_content='Start up your Jupyter notebook. \\nGo to the directory where you want your notebook and Hamilton function module(s) to live.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", " Document(page_content='Create a python file(s). Do that by going to “New > text file”. It’ll open a “file” editor view. Name the file and give it a .py extension. Once you save it, you’ll see that jupyter now provides python syntax highlighting. Keep this tab open, so you can', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", " Document(page_content='Keep this tab open, so you can flip back to it to edit this file.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", @@ -15053,9 +15053,9 @@ " 'target=\"_blank\">
    \\n'\n", - " ' \"Total\\n'\n", " '\\n'\n", " '

    Hamilton is a general-purpose framework to write '\n", @@ -19024,9 +19024,9 @@ " 'target=\"_blank\">\\n'\n", - " ' \"Total\\n'\n", " '\\n'\n", " '

    Hamilton is a general-purpose framework to write '\n", @@ -26175,7 +26175,7 @@ " '

    ** Overview **

    \\n'\n", " '

    This is part of the hamilton pyspark integration. To '\n", " 'install, run:

    \\n'\n", - " '

    pip install sf-hamilton[pyspark]

    \\n'\n", + " '

    pip install apache-hamilton[pyspark]

    \\n'\n", " '

    Reference Documentation

    \\n'\n", " '
    \\n'\n", " '
    \\n'\n", " '\\n'\n", " '',\n", - " 'chunks': [Document(page_content='** Overview ** \\nThis is part of the hamilton pyspark integration. To install, run: \\npip install sf-hamilton[pyspark] \\nReference Documentation', metadata={'Header 1': 'with_columns¶'}),\n", + " 'chunks': [Document(page_content='** Overview ** \\nThis is part of the hamilton pyspark integration. To install, run: \\npip install apache-hamilton[pyspark] \\nReference Documentation', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='groups of map operations on a dataframe, represented as pandas/primitives UDFs. This effectively “linearizes” compute – meaning that a DAG of map operations can be run as a set of .withColumn operations on a single dataframe – ensuring that you don’t have', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='– ensuring that you don’t have to do a complex extract then join process on spark, which can be inefficient.', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='Here’s an example of calling it – if you’ve seen @subdag, you should be familiar with the concepts:', metadata={'Header 1': 'with_columns¶'}),\n", @@ -32709,7 +32709,7 @@ " '
  • Publish your Hamilton functions in a package, or '\n", " 'import via other means (e.g. checkout a repository & '\n", " 'include in python path).

  • \\n'\n", - " '
  • Include sf-hamilton as a python '\n", + " '

  • Include apache-hamilton as a python '\n", " 'dependency

  • \\n'\n", " '
  • Have your ETL system execute your “driver” '\n", " 'code.

  • \\n'\n", @@ -32722,7 +32722,7 @@ " Document(page_content='Dagster \\n✅ \\nPrefect \\n✅ (see [prefect example](https://github.com/apache/hamilton/tree/main/examples/prefect)) \\nKubeflow Pipelines \\n✅ \\nCRON \\n✅ \\ndbt \\n✅ (see dbt example) \\nkubernetes', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", " Document(page_content='kubernetes \\n✅ but you need to setup kubernetes to run an image that can run python code - e.g. see Running a python application on kubernetes \\ndocker \\n✅ but you need to setup a docker image that can execute python code.', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", " Document(page_content='… in general if it runs python 3.7+ … \\n✅', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", - " Document(page_content='Write Hamilton functions & “driver” code. \\nPublish your Hamilton functions in a package, or import via other means (e.g. checkout a repository & include in python path). \\nInclude sf-hamilton as a python dependency', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'}),\n", + " Document(page_content='Write Hamilton functions & “driver” code. \\nPublish your Hamilton functions in a package, or import via other means (e.g. checkout a repository & include in python path). \\nInclude apache-hamilton as a python dependency', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'}),\n", " Document(page_content='Have your ETL system execute your “driver” code. \\nProfit.', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'})],\n", " 'url': 'https://hamilton.dagworks.io/en/latest/concepts/best-practices/using-within-your-etl-system/'}\n", "{'article_text': '
    \\n'\n", @@ -33983,7 +33983,7 @@ " 'you can install with pip install sf-hamilton[pandera]. Then, you '\n", + " 'class=\"pre\">apache-hamilton[pandera]. Then, you '\n", " 'can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).

    \\n'\n", @@ -34798,7 +34798,7 @@ " Document(page_content='(instead of functions). See hamilton/data_quality/default_validators.py or reach out on Slack for help!', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", " Document(page_content='Note \\nAs you see, validation steps effectively add nodes to the dataflow and the visualization. This helps trace which specific check failed for instance, but it can make visualizations harder to read.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", " Document(page_content='You can hide these nodes using the custom visualization style feature (see Apply custom style) by applying the style {\"style\": \"invis\"} to nodes with the tag hamilton.data_quality.source_node. This will only keep the original nodes and their _raw variant.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", - " Document(page_content='Hamilton has a pandera plugin for data validation that you can install with pip install sf-hamilton[pandera]. Then, you can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': 'pandera support¶'}),\n", + " Document(page_content='Hamilton has a pandera plugin for data validation that you can install with pip install apache-hamilton[pandera]. Then, you can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': 'pandera support¶'}),\n", " Document(page_content='Sometimes, your node outputs multiple values that you would like to name and make available to other nodes. These function modifiers act on the node output / function return. \\nNote', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶'}),\n", " Document(page_content='Note \\nTo add metadata to extracted nodes, use @tag_output, which works just like @tag.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶'}),\n", " Document(page_content='A good example is splitting a dataset into train, validation, and test splits. We will use @extract_fields, which requires specifying in a dictionary the field_name: field_type of each field.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶', 'Header 3': '@extract_fields¶'}),\n", @@ -37414,7 +37414,7 @@ " '
    pip '\n",
           "                 'install "sf-hamilton[sdk]"\\n'\n",
    +      "                 'class=\"s2\">"apache-hamilton[sdk]"\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", " '

    Then, navigate to the project page (dashboard/projects), '\n", @@ -37591,7 +37591,7 @@ " Document(page_content='Start the frontend server \\nThis takes a bit of time! So be patient. The server will be running on port 8242. Then navigate to http://localhost:8242 in your browser, and enter your email (this will be the username used within the app).', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", " Document(page_content='If building the Docker containers from scratch, increase your Docker memory to 10gb or more – you can do this in the Docker Desktop settings. \\nTo build the images locally, you can run the following command:', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", " Document(page_content='# from the hamilton/ui directory ./deployment/dev.sh --build \\nThis will build the containers from scratch. If you just want to mount the local code, you can run just \\n./deployment/dev.sh', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", - " Document(page_content='Now that you have your server running, you can run a simple dataflow and watch it in the UI! You can follow instructions in the UI when you create a new project, or follow the instructions here. \\nFirst, install the SDK: \\npip install \"sf-hamilton[sdk]\"', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", + " Document(page_content='Now that you have your server running, you can run a simple dataflow and watch it in the UI! You can follow instructions in the UI when you create a new project, or follow the instructions here. \\nFirst, install the SDK: \\npip install \"apache-hamilton[sdk]\"', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='Then, navigate to the project page (dashboard/projects), in the running UI, and click the green + New DAG button. \\nRemember the project ID – you’ll use it for the next steps. \\nAdd the following adapter to your code if you have existing Hamilton code:', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='from hamilton_sdk import adapters tracker = adapters.HamiltonTracker( project_id=PROJECT_ID_FROM_ABOVE, username=\"EMAIL_YOU_PUT_IN_THE_UI\", dag_name=\"my_version_of_the_dag\", tags={\"environment\": \"DEV\", \"team\": \"MY_TEAM\", \"version\": \"X\"} ) dr = (', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='\"version\": \"X\"} ) dr = ( driver.Builder() .with_config(your_config) .with_modules(*your_modules) .with_adapters(tracker) .build() )', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", @@ -38278,18 +38278,18 @@ " 'href=\"#installing-with-pip\" title=\"Link to this '\n", " 'heading\">¶\\n'\n", " '

    Hamilton is published on pypi under '\n", + " 'href=\"https://pypi.org/project/apache-hamilton/\">pypi under '\n", " 'sf-hamilton. To install, run:

    \\n'\n", + " 'class=\"pre\">apache-hamilton. To install, run:

    \\n'\n", " '

    pip install '\n", - " 'sf-hamilton

    \\n'\n", + " 'apache-hamilton

    \\n'\n", " '

    To use the DAG visualization functionality, instead '\n", " 'install with

    \\n'\n", " '

    pip install '\n", " 'sf-hamilton[visualization]

    \\n'\n", + " 'class=\"pre\">apache-hamilton[visualization]

    \\n'\n", " '

    Note: for visualization you may additionally need to '\n", " 'install graphviz externally – see\\n'\n", " 'conda install '\n", " '-c hamilton-opensource sf-hamilton

    \\n'\n", + " 'class=\"pre\">apache-hamilton

    \\n'\n", " '
    \\n'\n", " '
    \\n'\n", " '

    Installing from source',\n", " 'chunks': [Document(page_content='Installing hamilton is easy!'),\n", " Document(page_content='Hamilton is a lightweight framework with a variety of extensions/plugins. To get started, you’ll need the following: \\npython >= 3.8 \\npip \\nFor help with python/pip/managing virtual environments see the python docs.', metadata={'Header 1': 'Install¶'}),\n", - " Document(page_content='Hamilton is published on pypi under sf-hamilton. To install, run: \\npip install sf-hamilton \\nTo use the DAG visualization functionality, instead install with \\npip install sf-hamilton[visualization]', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", + " Document(page_content='Hamilton is published on pypi under apache-hamilton. To install, run: \\npip install apache-hamilton \\nTo use the DAG visualization functionality, instead install with \\npip install apache-hamilton[visualization]', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", " Document(page_content='Note: for visualization you may additionally need to install graphviz externally – see graphviz for instructions on the correct way for your operating system.', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", - " Document(page_content='Hamilton is also available on conda if you prefer: \\nconda install -c hamilton-opensource sf-hamilton', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with conda¶'}),\n", + " Document(page_content='Hamilton is also available on conda if you prefer: \\nconda install -c hamilton-opensource apache-hamilton', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with conda¶'}),\n", " Document(page_content='You can also download the code and run it from the source. \\ngit clone https://github.com/apache/hamilton.git cd hamilton pip install -e .', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing from source¶'})],\n", " 'url': 'https://hamilton.dagworks.io/en/latest/get-started/install/'}\n", "{'article_text': '
    \\n'\n", @@ -39436,7 +39436,7 @@ " 'inputs=initial_columns)\\n'\n", " ' # `pip install '\n", - " 'sf-hamilton[visualization]` earlier you can also do\\n'\n", + " 'apache-hamilton[visualization]` earlier you can also do\\n'\n", " ' # '\n", " 'dr.visualize_execution(output_columns,'./my_dag.dot', '\n", " '{})\\n'\n", @@ -39517,7 +39517,7 @@ " Document(page_content='# Instantiate a common spine for your pipeline index = pd.date_range(\"2022-01-01\", periods=6, freq=\"w\") initial_columns = { # load from actuals or wherever -- this is our initial data we use as input. # Note: these do not have to be all series, they could', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"to be all series, they could be scalar inputs. 'signups': pd.Series([1, 10, 50, 100, 200, 400], index=index), 'spend': pd.Series([10, 10, 20, 40, 40, 50], index=index), } # we need to tell hamilton where to load function definitions from config = {} # we\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"from config = {} # we don't have any configuration or invariant data for this example. dr = driver.Driver(config, my_functions) # can pass in multiple modules # we need to specify what we want in the final dataframe. output_columns = [ 'spend', 'signups',\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", - " Document(page_content=\"= [ 'spend', 'signups', 'avg_3wk_spend', 'acquisition_cost', ] # let's create the dataframe! df = dr.execute(output_columns, inputs=initial_columns) # `pip install sf-hamilton[visualization]` earlier you can also do #\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", + " Document(page_content=\"= [ 'spend', 'signups', 'avg_3wk_spend', 'acquisition_cost', ] # let's create the dataframe! df = dr.execute(output_columns, inputs=initial_columns) # `pip install apache-hamilton[visualization]` earlier you can also do #\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"earlier you can also do # dr.visualize_execution(output_columns,'./my_dag.dot', {}) print(df)\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content='Run the script with the following command: \\npython my_script.py \\nAnd you should see the following output:', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content='spend signups avg_3wk_spend acquisition_cost 2022-01-02 10 1 NaN 10.000 2022-01-09 10 10 NaN 1.000 2022-01-16 20 50 13.333333 0.400 2022-01-23 40 100 23.333333 0.400 2022-01-30 40 200 33.333333 0.200 2022-02-06 50 400 43.333333 0.125', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", @@ -39562,7 +39562,7 @@ " 'via

    \\n'\n", " '
    pip '\n",
    -      "                 'install sf-hamilton[cli]\\n'\n",
    +      "                 'install apache-hamilton[cli]\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", " '

    You can verify the installation with

    \\n'\n", @@ -39741,7 +39741,7 @@ " '
    \\n'\n", " '

    ',\n", " 'chunks': [Document(page_content='This page covers the Hamilton CLI. It is built directly from the CLI, but note that the command hamilton --help always provide the most accurate documentation.', metadata={'Header 1': 'Command line interface¶'}),\n", - " Document(page_content='The dependencies for the Hamilton CLI can be installed via \\npip install sf-hamilton[cli] \\nYou can verify the installation with \\nhamilton --help', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'Installation¶'}),\n", + " Document(page_content='The dependencies for the Hamilton CLI can be installed via \\npip install apache-hamilton[cli] \\nYou can verify the installation with \\nhamilton --help', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'Installation¶'}),\n", " Document(page_content='Options: \\n--verbose / --no-verbose: [default: no-verbose] \\n--json-out / --no-json-out: [default: no-json-out] \\n--install-completion: Install completion for the current shell.', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", " Document(page_content='--show-completion: Show completion for the current shell, to copy it or customize the installation. \\n--help: Show this message and exit. \\nCommands: \\nbuild: Build a single Driver with MODULES \\ndiff: Diff between the current MODULES and their…', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", " Document(page_content='version: Version NODES and DATAFLOW from dataflow… \\nview: Build and visualize dataflow with MODULES', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", @@ -40277,7 +40277,7 @@ " 'class=\"pre\">--help messages.

    \\n'\n", " '
    pip '\n",
    -      "                 'install sf-hamilton[cli]\\n'\n",
    +      "                 'install apache-hamilton[cli]\\n'\n",
           "                 'hamilton --help\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", @@ -40545,7 +40545,7 @@ " Document(page_content='install the hooks defined in .pre-commit-config.yaml \\npre-commit install \\nNow, hooks will automatically run on git commit \\nto manually run hooks \\npre-commit run --all-files', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Add pre-commit hooks to your project¶', 'Header 3': 'Steps to get started¶'}),\n", " Document(page_content='pre-commit hooks are great developer tools, but off-the-shelf solutions aren’t aware of the Hamilton framework. Hence, we developed a pre-commit hook to help you author Hamilton dataflows! Under the hood, they leverage the hamilton CLI, so if you are', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", " Document(page_content='the hamilton CLI, so if you are unfamiliar with it, feel free to install it and view the --help messages.', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", - " Document(page_content='pip install sf-hamilton[cli] hamilton --help', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", + " Document(page_content='pip install apache-hamilton[cli] hamilton --help', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", " Document(page_content='Hamilton doesn’t have many syntactic constraints, but there’s a few things we want to catch: \\nfunctions parameters and return are type annotated \\na node consistently has the same type (e.g., a parameter in multiple functions)', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", " Document(page_content='functions with a name starting with underscore (_) are ignored from the dataflow \\nfunctions with a @config decorator received a trailing double underscore with a suffix (e.g., hello__weekday(), hello__weekend())', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", " Document(page_content='Instead of reimplementing this logic, we can try to build the Hamilton Driver with the command hamilton build MODULES and catch errors. This also ensures the verification is always in sync with the actual build mechanism. This hook will help prevent us', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", @@ -41815,7 +41815,7 @@ " 'class=\"highlight\">
    pip '\n",
           "                 'install jupyterlab\\n'\n",
           "                 'pip install '\n",
    -      "                 'sf-hamilton\\n'\n",
    +      "                 'apache-hamilton\\n'\n",
           "                 '
    \\n'\n", " '\\n'\n", " '

    Then to start the notebook server it should just be:

    \\n'\n", @@ -42139,7 +42139,7 @@ " Document(page_content=\"# Step 3 - add the module to the driver and continue as usual dr = driver.Driver(config, some_functions, temp_module) df = dr.execute(['avg_3wk_spend', 'log_avg_3wk_spend'], inputs=input_data)\", metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '1 - Dynamically create modules within your notebook¶', 'Header 3': 'Using ad_hoc_utils to create a temporary module (e.g. use in google colab)¶'}),\n", " Document(page_content='Using a “temporary python module” will not enable scaling of computation by using Ray, Dask, or Pandas on Spark. So we suggest only using this approach for development purposes only.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '1 - Dynamically create modules within your notebook¶', 'Header 3': 'Caveat with this approach:¶'}),\n", " Document(page_content='This tutorial can also be found published on TDS.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶'}),\n", - " Document(page_content='I assume you already have this step set up. But just in case you don’t: \\npip install jupyterlab pip install sf-hamilton \\nThen to start the notebook server it should just be:', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 1 — Install Jupyter & Hamilton ¶'}),\n", + " Document(page_content='I assume you already have this step set up. But just in case you don’t: \\npip install jupyterlab pip install apache-hamilton \\nThen to start the notebook server it should just be:', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 1 — Install Jupyter & Hamilton ¶'}),\n", " Document(page_content='Start up your Jupyter notebook. \\nGo to the directory where you want your notebook and Hamilton function module(s) to live.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", " Document(page_content='Create a python file(s). Do that by going to “New > text file”. It’ll open a “file” editor view. Name the file and give it a .py extension. Once you save it, you’ll see that jupyter now provides python syntax highlighting. Keep this tab open, so you can', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", " Document(page_content='Keep this tab open, so you can flip back to it to edit this file.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", @@ -42260,9 +42260,9 @@ " 'target=\"_blank\">
    \\n'\n", - " ' \"Total\\n'\n", " '\\n'\n", " '

    Hamilton is a general-purpose framework to write '\n", @@ -46231,9 +46231,9 @@ " 'target=\"_blank\">\\n'\n", - " ' \"Total\\n'\n", " '\\n'\n", " '

    Hamilton is a general-purpose framework to write '\n", @@ -53382,7 +53382,7 @@ " '

    ** Overview **

    \\n'\n", " '

    This is part of the hamilton pyspark integration. To '\n", " 'install, run:

    \\n'\n", - " '

    pip install sf-hamilton[pyspark]

    \\n'\n", + " '

    pip install apache-hamilton[pyspark]

    \\n'\n", " '

    Reference Documentation

    \\n'\n", " '
    \\n'\n", " '
    \\n'\n", " '\\n'\n", " '',\n", - " 'chunks': [Document(page_content='** Overview ** \\nThis is part of the hamilton pyspark integration. To install, run: \\npip install sf-hamilton[pyspark] \\nReference Documentation', metadata={'Header 1': 'with_columns¶'}),\n", + " 'chunks': [Document(page_content='** Overview ** \\nThis is part of the hamilton pyspark integration. To install, run: \\npip install apache-hamilton[pyspark] \\nReference Documentation', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='groups of map operations on a dataframe, represented as pandas/primitives UDFs. This effectively “linearizes” compute – meaning that a DAG of map operations can be run as a set of .withColumn operations on a single dataframe – ensuring that you don’t have', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='– ensuring that you don’t have to do a complex extract then join process on spark, which can be inefficient.', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='Here’s an example of calling it – if you’ve seen @subdag, you should be familiar with the concepts:', metadata={'Header 1': 'with_columns¶'}),\n", @@ -59916,7 +59916,7 @@ " '
  • Publish your Hamilton functions in a package, or '\n", " 'import via other means (e.g. checkout a repository & '\n", " 'include in python path).

  • \\n'\n", - " '
  • Include sf-hamilton as a python '\n", + " '

  • Include apache-hamilton as a python '\n", " 'dependency

  • \\n'\n", " '
  • Have your ETL system execute your “driver” '\n", " 'code.

  • \\n'\n", @@ -59929,7 +59929,7 @@ " Document(page_content='Dagster \\n✅ \\nPrefect \\n✅ (see [prefect example](https://github.com/apache/hamilton/tree/main/examples/prefect)) \\nKubeflow Pipelines \\n✅ \\nCRON \\n✅ \\ndbt \\n✅ (see dbt example) \\nkubernetes', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", " Document(page_content='kubernetes \\n✅ but you need to setup kubernetes to run an image that can run python code - e.g. see Running a python application on kubernetes \\ndocker \\n✅ but you need to setup a docker image that can execute python code.', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", " Document(page_content='… in general if it runs python 3.7+ … \\n✅', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'Compatibility Matrix¶'}),\n", - " Document(page_content='Write Hamilton functions & “driver” code. \\nPublish your Hamilton functions in a package, or import via other means (e.g. checkout a repository & include in python path). \\nInclude sf-hamilton as a python dependency', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'}),\n", + " Document(page_content='Write Hamilton functions & “driver” code. \\nPublish your Hamilton functions in a package, or import via other means (e.g. checkout a repository & include in python path). \\nInclude apache-hamilton as a python dependency', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'}),\n", " Document(page_content='Have your ETL system execute your “driver” code. \\nProfit.', metadata={'Header 1': 'Using within your ETL System¶', 'Header 2': 'ETL Recipe¶'})],\n", " 'url': 'https://hamilton.dagworks.io/en/latest/concepts/best-practices/using-within-your-etl-system/'}\n", "{'article_text': '
    \\n'\n", @@ -61190,7 +61190,7 @@ " 'you can install with pip install sf-hamilton[pandera]. Then, you '\n", + " 'class=\"pre\">apache-hamilton[pandera]. Then, you '\n", " 'can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).

    \\n'\n", @@ -62005,7 +62005,7 @@ " Document(page_content='(instead of functions). See hamilton/data_quality/default_validators.py or reach out on Slack for help!', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", " Document(page_content='Note \\nAs you see, validation steps effectively add nodes to the dataflow and the visualization. This helps trace which specific check failed for instance, but it can make visualizations harder to read.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", " Document(page_content='You can hide these nodes using the custom visualization style feature (see Apply custom style) by applying the style {\"style\": \"invis\"} to nodes with the tag hamilton.data_quality.source_node. This will only keep the original nodes and their _raw variant.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': '@check_output¶'}),\n", - " Document(page_content='Hamilton has a pandera plugin for data validation that you can install with pip install sf-hamilton[pandera]. Then, you can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': 'pandera support¶'}),\n", + " Document(page_content='Hamilton has a pandera plugin for data validation that you can install with pip install apache-hamilton[pandera]. Then, you can pass a pandera schema (for DataFrame or Series) to @check_output(schema=...).', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Validate node output¶', 'Header 3': 'pandera support¶'}),\n", " Document(page_content='Sometimes, your node outputs multiple values that you would like to name and make available to other nodes. These function modifiers act on the node output / function return. \\nNote', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶'}),\n", " Document(page_content='Note \\nTo add metadata to extracted nodes, use @tag_output, which works just like @tag.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶'}),\n", " Document(page_content='A good example is splitting a dataset into train, validation, and test splits. We will use @extract_fields, which requires specifying in a dictionary the field_name: field_type of each field.', metadata={'Header 1': 'Function modifiers¶', 'Header 2': 'Split node output into n nodes¶', 'Header 3': '@extract_fields¶'}),\n", @@ -64621,7 +64621,7 @@ " '
    pip '\n",
           "                 'install "sf-hamilton[sdk]"\\n'\n",
    +      "                 'class=\"s2\">"apache-hamilton[sdk]"\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", " '

    Then, navigate to the project page (dashboard/projects), '\n", @@ -64798,7 +64798,7 @@ " Document(page_content='Start the frontend server \\nThis takes a bit of time! So be patient. The server will be running on port 8242. Then navigate to http://localhost:8242 in your browser, and enter your email (this will be the username used within the app).', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", " Document(page_content='If building the Docker containers from scratch, increase your Docker memory to 10gb or more – you can do this in the Docker Desktop settings. \\nTo build the images locally, you can run the following command:', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", " Document(page_content='# from the hamilton/ui directory ./deployment/dev.sh --build \\nThis will build the containers from scratch. If you just want to mount the local code, you can run just \\n./deployment/dev.sh', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Starting the UI¶'}),\n", - " Document(page_content='Now that you have your server running, you can run a simple dataflow and watch it in the UI! You can follow instructions in the UI when you create a new project, or follow the instructions here. \\nFirst, install the SDK: \\npip install \"sf-hamilton[sdk]\"', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", + " Document(page_content='Now that you have your server running, you can run a simple dataflow and watch it in the UI! You can follow instructions in the UI when you create a new project, or follow the instructions here. \\nFirst, install the SDK: \\npip install \"apache-hamilton[sdk]\"', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='Then, navigate to the project page (dashboard/projects), in the running UI, and click the green + New DAG button. \\nRemember the project ID – you’ll use it for the next steps. \\nAdd the following adapter to your code if you have existing Hamilton code:', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='from hamilton_sdk import adapters tracker = adapters.HamiltonTracker( project_id=PROJECT_ID_FROM_ABOVE, username=\"EMAIL_YOU_PUT_IN_THE_UI\", dag_name=\"my_version_of_the_dag\", tags={\"environment\": \"DEV\", \"team\": \"MY_TEAM\", \"version\": \"X\"} ) dr = (', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", " Document(page_content='\"version\": \"X\"} ) dr = ( driver.Builder() .with_config(your_config) .with_modules(*your_modules) .with_adapters(tracker) .build() )', metadata={'Header 1': 'Hamilton UI¶', 'Header 2': 'Overview¶', 'Header 3': 'Running your first dataflows¶'}),\n", @@ -65485,18 +65485,18 @@ " 'href=\"#installing-with-pip\" title=\"Link to this '\n", " 'heading\">¶\\n'\n", " '

    Hamilton is published on pypi under '\n", + " 'href=\"https://pypi.org/project/apache-hamilton/\">pypi under '\n", " 'sf-hamilton. To install, run:

    \\n'\n", + " 'class=\"pre\">apache-hamilton. To install, run:

    \\n'\n", " '

    pip install '\n", - " 'sf-hamilton

    \\n'\n", + " 'apache-hamilton

    \\n'\n", " '

    To use the DAG visualization functionality, instead '\n", " 'install with

    \\n'\n", " '

    pip install '\n", " 'sf-hamilton[visualization]

    \\n'\n", + " 'class=\"pre\">apache-hamilton[visualization]

    \\n'\n", " '

    Note: for visualization you may additionally need to '\n", " 'install graphviz externally – see\\n'\n", " 'conda install '\n", " '-c hamilton-opensource sf-hamilton

    \\n'\n", + " 'class=\"pre\">apache-hamilton

    \\n'\n", " '
    \\n'\n", " '
    \\n'\n", " '

    Installing from source',\n", " 'chunks': [Document(page_content='Installing hamilton is easy!'),\n", " Document(page_content='Hamilton is a lightweight framework with a variety of extensions/plugins. To get started, you’ll need the following: \\npython >= 3.8 \\npip \\nFor help with python/pip/managing virtual environments see the python docs.', metadata={'Header 1': 'Install¶'}),\n", - " Document(page_content='Hamilton is published on pypi under sf-hamilton. To install, run: \\npip install sf-hamilton \\nTo use the DAG visualization functionality, instead install with \\npip install sf-hamilton[visualization]', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", + " Document(page_content='Hamilton is published on pypi under apache-hamilton. To install, run: \\npip install apache-hamilton \\nTo use the DAG visualization functionality, instead install with \\npip install apache-hamilton[visualization]', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", " Document(page_content='Note: for visualization you may additionally need to install graphviz externally – see graphviz for instructions on the correct way for your operating system.', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with pip¶'}),\n", - " Document(page_content='Hamilton is also available on conda if you prefer: \\nconda install -c hamilton-opensource sf-hamilton', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with conda¶'}),\n", + " Document(page_content='Hamilton is also available on conda if you prefer: \\nconda install -c hamilton-opensource apache-hamilton', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing with conda¶'}),\n", " Document(page_content='You can also download the code and run it from the source. \\ngit clone https://github.com/apache/hamilton.git cd hamilton pip install -e .', metadata={'Header 1': 'Install¶', 'Header 2': 'Installing from source¶'})],\n", " 'url': 'https://hamilton.dagworks.io/en/latest/get-started/install/'}\n", "{'article_text': '
    \\n'\n", @@ -66643,7 +66643,7 @@ " 'inputs=initial_columns)\\n'\n", " ' # `pip install '\n", - " 'sf-hamilton[visualization]` earlier you can also do\\n'\n", + " 'apache-hamilton[visualization]` earlier you can also do\\n'\n", " ' # '\n", " 'dr.visualize_execution(output_columns,'./my_dag.dot', '\n", " '{})\\n'\n", @@ -66724,7 +66724,7 @@ " Document(page_content='# Instantiate a common spine for your pipeline index = pd.date_range(\"2022-01-01\", periods=6, freq=\"w\") initial_columns = { # load from actuals or wherever -- this is our initial data we use as input. # Note: these do not have to be all series, they could', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"to be all series, they could be scalar inputs. 'signups': pd.Series([1, 10, 50, 100, 200, 400], index=index), 'spend': pd.Series([10, 10, 20, 40, 40, 50], index=index), } # we need to tell hamilton where to load function definitions from config = {} # we\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"from config = {} # we don't have any configuration or invariant data for this example. dr = driver.Driver(config, my_functions) # can pass in multiple modules # we need to specify what we want in the final dataframe. output_columns = [ 'spend', 'signups',\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", - " Document(page_content=\"= [ 'spend', 'signups', 'avg_3wk_spend', 'acquisition_cost', ] # let's create the dataframe! df = dr.execute(output_columns, inputs=initial_columns) # `pip install sf-hamilton[visualization]` earlier you can also do #\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", + " Document(page_content=\"= [ 'spend', 'signups', 'avg_3wk_spend', 'acquisition_cost', ] # let's create the dataframe! df = dr.execute(output_columns, inputs=initial_columns) # `pip install apache-hamilton[visualization]` earlier you can also do #\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content=\"earlier you can also do # dr.visualize_execution(output_columns,'./my_dag.dot', {}) print(df)\", metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content='Run the script with the following command: \\npython my_script.py \\nAnd you should see the following output:', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", " Document(page_content='spend signups avg_3wk_spend acquisition_cost 2022-01-02 10 1 NaN 10.000 2022-01-09 10 10 NaN 1.000 2022-01-16 20 50 13.333333 0.400 2022-01-23 40 100 23.333333 0.400 2022-01-30 40 200 33.333333 0.200 2022-02-06 50 400 43.333333 0.125', metadata={'Header 1': 'Your First Dataflow¶', 'Header 2': 'Run your dataflow¶'}),\n", @@ -66769,7 +66769,7 @@ " 'via

    \\n'\n", " '
    pip '\n",
    -      "                 'install sf-hamilton[cli]\\n'\n",
    +      "                 'install apache-hamilton[cli]\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", " '

    You can verify the installation with

    \\n'\n", @@ -66948,7 +66948,7 @@ " '
    \\n'\n", " '

    ',\n", " 'chunks': [Document(page_content='This page covers the Hamilton CLI. It is built directly from the CLI, but note that the command hamilton --help always provide the most accurate documentation.', metadata={'Header 1': 'Command line interface¶'}),\n", - " Document(page_content='The dependencies for the Hamilton CLI can be installed via \\npip install sf-hamilton[cli] \\nYou can verify the installation with \\nhamilton --help', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'Installation¶'}),\n", + " Document(page_content='The dependencies for the Hamilton CLI can be installed via \\npip install apache-hamilton[cli] \\nYou can verify the installation with \\nhamilton --help', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'Installation¶'}),\n", " Document(page_content='Options: \\n--verbose / --no-verbose: [default: no-verbose] \\n--json-out / --no-json-out: [default: no-json-out] \\n--install-completion: Install completion for the current shell.', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", " Document(page_content='--show-completion: Show completion for the current shell, to copy it or customize the installation. \\n--help: Show this message and exit. \\nCommands: \\nbuild: Build a single Driver with MODULES \\ndiff: Diff between the current MODULES and their…', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", " Document(page_content='version: Version NODES and DATAFLOW from dataflow… \\nview: Build and visualize dataflow with MODULES', metadata={'Header 1': 'Command line interface¶', 'Header 2': 'hamilton (global)¶'}),\n", @@ -67484,7 +67484,7 @@ " 'class=\"pre\">--help messages.

    \\n'\n", " '
    pip '\n",
    -      "                 'install sf-hamilton[cli]\\n'\n",
    +      "                 'install apache-hamilton[cli]\\n'\n",
           "                 'hamilton --help\\n'\n",
           "                 '
    \\n'\n", " '
    \\n'\n", @@ -67752,7 +67752,7 @@ " Document(page_content='install the hooks defined in .pre-commit-config.yaml \\npre-commit install \\nNow, hooks will automatically run on git commit \\nto manually run hooks \\npre-commit run --all-files', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Add pre-commit hooks to your project¶', 'Header 3': 'Steps to get started¶'}),\n", " Document(page_content='pre-commit hooks are great developer tools, but off-the-shelf solutions aren’t aware of the Hamilton framework. Hence, we developed a pre-commit hook to help you author Hamilton dataflows! Under the hood, they leverage the hamilton CLI, so if you are', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", " Document(page_content='the hamilton CLI, so if you are unfamiliar with it, feel free to install it and view the --help messages.', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", - " Document(page_content='pip install sf-hamilton[cli] hamilton --help', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", + " Document(page_content='pip install apache-hamilton[cli] hamilton --help', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶'}),\n", " Document(page_content='Hamilton doesn’t have many syntactic constraints, but there’s a few things we want to catch: \\nfunctions parameters and return are type annotated \\na node consistently has the same type (e.g., a parameter in multiple functions)', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", " Document(page_content='functions with a name starting with underscore (_) are ignored from the dataflow \\nfunctions with a @config decorator received a trailing double underscore with a suffix (e.g., hello__weekday(), hello__weekend())', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", " Document(page_content='Instead of reimplementing this logic, we can try to build the Hamilton Driver with the command hamilton build MODULES and catch errors. This also ensures the verification is always in sync with the actual build mechanism. This hook will help prevent us', metadata={'Header 1': 'pre-commit hooks¶', 'Header 2': 'Custom Hamilton pre-commit hooks¶', 'Header 3': 'Checking dataflow definition¶'}),\n", @@ -69022,7 +69022,7 @@ " 'class=\"highlight\">
    pip '\n",
           "                 'install jupyterlab\\n'\n",
           "                 'pip install '\n",
    -      "                 'sf-hamilton\\n'\n",
    +      "                 'apache-hamilton\\n'\n",
           "                 '
    \\n'\n", " '\\n'\n", " '

    Then to start the notebook server it should just be:

    \\n'\n", @@ -69346,7 +69346,7 @@ " Document(page_content=\"# Step 3 - add the module to the driver and continue as usual dr = driver.Driver(config, some_functions, temp_module) df = dr.execute(['avg_3wk_spend', 'log_avg_3wk_spend'], inputs=input_data)\", metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '1 - Dynamically create modules within your notebook¶', 'Header 3': 'Using ad_hoc_utils to create a temporary module (e.g. use in google colab)¶'}),\n", " Document(page_content='Using a “temporary python module” will not enable scaling of computation by using Ray, Dask, or Pandas on Spark. So we suggest only using this approach for development purposes only.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '1 - Dynamically create modules within your notebook¶', 'Header 3': 'Caveat with this approach:¶'}),\n", " Document(page_content='This tutorial can also be found published on TDS.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶'}),\n", - " Document(page_content='I assume you already have this step set up. But just in case you don’t: \\npip install jupyterlab pip install sf-hamilton \\nThen to start the notebook server it should just be:', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 1 — Install Jupyter & Hamilton ¶'}),\n", + " Document(page_content='I assume you already have this step set up. But just in case you don’t: \\npip install jupyterlab pip install apache-hamilton \\nThen to start the notebook server it should just be:', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 1 — Install Jupyter & Hamilton ¶'}),\n", " Document(page_content='Start up your Jupyter notebook. \\nGo to the directory where you want your notebook and Hamilton function module(s) to live.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", " Document(page_content='Create a python file(s). Do that by going to “New > text file”. It’ll open a “file” editor view. Name the file and give it a .py extension. Once you save it, you’ll see that jupyter now provides python syntax highlighting. Keep this tab open, so you can', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", " Document(page_content='Keep this tab open, so you can flip back to it to edit this file.', metadata={'Header 1': 'Jupyter notebooks¶', 'Header 2': '2 - Importing modules into your notebook¶', 'Header 3': 'Step 2— Set up the files ¶'}),\n", @@ -69467,9 +69467,9 @@ " 'target=\"_blank\">
    \\n'\n", - " ' \"Total\\n'\n", " '\\n'\n", " '

    Hamilton is a general-purpose framework to write '\n", @@ -73438,9 +73438,9 @@ " 'target=\"_blank\">\\n'\n", - " ' \"Total\\n'\n", " '\\n'\n", " '

    Hamilton is a general-purpose framework to write '\n", @@ -80589,7 +80589,7 @@ " '

    ** Overview **

    \\n'\n", " '

    This is part of the hamilton pyspark integration. To '\n", " 'install, run:

    \\n'\n", - " '

    pip install sf-hamilton[pyspark]

    \\n'\n", + " '

    pip install apache-hamilton[pyspark]

    \\n'\n", " '

    Reference Documentation

    \\n'\n", " '
    \\n'\n", " '
    \\n'\n", " '\\n'\n", " '',\n", - " 'chunks': [Document(page_content='** Overview ** \\nThis is part of the hamilton pyspark integration. To install, run: \\npip install sf-hamilton[pyspark] \\nReference Documentation', metadata={'Header 1': 'with_columns¶'}),\n", + " 'chunks': [Document(page_content='** Overview ** \\nThis is part of the hamilton pyspark integration. To install, run: \\npip install apache-hamilton[pyspark] \\nReference Documentation', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='groups of map operations on a dataframe, represented as pandas/primitives UDFs. This effectively “linearizes” compute – meaning that a DAG of map operations can be run as a set of .withColumn operations on a single dataframe – ensuring that you don’t have', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='– ensuring that you don’t have to do a complex extract then join process on spark, which can be inefficient.', metadata={'Header 1': 'with_columns¶'}),\n", " Document(page_content='Here’s an example of calling it – if you’ve seen @subdag, you should be familiar with the concepts:', metadata={'Header 1': 'with_columns¶'}),\n", @@ -87342,7 +87342,7 @@ " '

    This walks the graph and translates it to run onto Dask.

    \\n'\n", - " '

    Use pip install sf-hamilton[dask] to get the '\n", + " '

    Use pip install apache-hamilton[dask] to get the '\n", " 'dependencies required to run this.

    \\n'\n", " '

    Try this adapter when:

    \\n'\n", " '
    \\n'\n", @@ -87933,7 +87933,7 @@ " '\\n'\n", " '\\n'\n", " '',\n", - " 'chunks': [Document(page_content='Runs the entire Hamilton DAG on dask. \\nClass representing what’s required to make Hamilton run on Dask. \\nThis walks the graph and translates it to run onto Dask. \\nUse pip install sf-hamilton[dask] to get the dependencies required to run this.', metadata={'Header 1': 'h_dask.DaskGraphAdapter¶'}),\n", + " 'chunks': [Document(page_content='Runs the entire Hamilton DAG on dask. \\nClass representing what’s required to make Hamilton run on Dask. \\nThis walks the graph and translates it to run onto Dask. \\nUse pip install apache-hamilton[dask] to get the dependencies required to run this.', metadata={'Header 1': 'h_dask.DaskGraphAdapter¶'}),\n", " Document(page_content='Try this adapter when:', metadata={'Header 1': 'h_dask.DaskGraphAdapter¶'}),\n", " Document(page_content='Dask is a good choice to scale computation when you really can’t do things in memory anymore with pandas. For most simple pandas operations, you should not have to do anything to scale! You just need to load in data via dask rather than pandas.', metadata={'Header 1': 'h_dask.DaskGraphAdapter¶'}),\n", " Document(page_content='Dask can help scale to larger data sets if running on a cluster – you’ll just have to switch to natively using their object types if that’s the case (set use_delayed=False, and compute_at_end=False).', metadata={'Header 1': 'h_dask.DaskGraphAdapter¶'}),\n", @@ -88220,7 +88220,7 @@ " '

    This walks the graph and translates it to run onto Ray.

    \\n'\n", - " '

    Use pip install sf-hamilton[ray] to get the '\n", + " '

    Use pip install apache-hamilton[ray] to get the '\n", " 'dependencies required to run this.

    \\n'\n", " '

    Use this if:

    \\n'\n", " '
    \\n'\n", @@ -88697,7 +88697,7 @@ " '\\n'\n", " '',\n", " 'chunks': [Document(page_content='The graph adapter to delegate execution of the individual nodes in a Hamilton graph to Ray. \\nClass representing what’s required to make Hamilton run on Ray. \\nThis walks the graph and translates it to run onto Ray.', metadata={'Header 1': 'h_ray.RayGraphAdapter¶'}),\n", - " Document(page_content='Use pip install sf-hamilton[ray] to get the dependencies required to run this. \\nUse this if:', metadata={'Header 1': 'h_ray.RayGraphAdapter¶'}),\n", + " Document(page_content='Use pip install apache-hamilton[ray] to get the dependencies required to run this. \\nUse this if:', metadata={'Header 1': 'h_ray.RayGraphAdapter¶'}),\n", " Document(page_content='you want to utilize multiple cores on a single machine, or you want to scale to larger data set sizes with a Ray cluster that you can connect to. Note (1): you are still constrained by machine memory size with Ray; you can’t just scale to any dataset', metadata={'Header 1': 'h_ray.RayGraphAdapter¶'}),\n", " Document(page_content='can’t just scale to any dataset size. Note (2): serialization costs can outweigh the benefits of parallelism so you should benchmark your code to see if it’s worth it.', metadata={'Header 1': 'h_ray.RayGraphAdapter¶'}),\n", " Document(page_content='Multi-core on single machine ✅ \\nDistributed computation on a Ray cluster ✅ \\nScales to any size of data ⛔️; you are LIMITED by the memory on the instance/computer 💻.', metadata={'Header 1': 'h_ray.RayGraphAdapter¶', 'Header 2': 'Notes on scaling:¶'}),\n", @@ -88750,7 +88750,7 @@ " 'title=\"Link to this definition\">¶\\n'\n", " '

    Class representing what’s required to make Hamilton '\n", " 'run Ray Workflows

    \\n'\n", - " '

    Use pip install sf-hamilton[ray] to get the '\n", + " '

    Use pip install apache-hamilton[ray] to get the '\n", " 'dependencies required to run this.

    \\n'\n", " '

    Ray workflows is a more robust way to scale computation '\n", " 'for any type of Hamilton graph.

    \\n'\n", @@ -89256,7 +89256,7 @@ " '
    \\n'\n", " '\\n'\n", " '',\n", - " 'chunks': [Document(page_content='A Graph Adapter for delegating the execution of hamilton nodes to Ray. \\nClass representing what’s required to make Hamilton run Ray Workflows \\nUse pip install sf-hamilton[ray] to get the dependencies required to run this.', metadata={'Header 1': 'h_ray.RayWorkflowGraphAdapter¶'}),\n", + " 'chunks': [Document(page_content='A Graph Adapter for delegating the execution of hamilton nodes to Ray. \\nClass representing what’s required to make Hamilton run Ray Workflows \\nUse pip install apache-hamilton[ray] to get the dependencies required to run this.', metadata={'Header 1': 'h_ray.RayWorkflowGraphAdapter¶'}),\n", " Document(page_content='Ray workflows is a more robust way to scale computation for any type of Hamilton graph.', metadata={'Header 1': 'h_ray.RayWorkflowGraphAdapter¶'}),\n", " Document(page_content='Ray workflows offer durable computation. That is, they save and checkpoint each function. \\nThis enables one to run a workflow, and not have to restart it if something fails, assuming correct Ray workflow usage.', metadata={'Header 1': 'h_ray.RayWorkflowGraphAdapter¶', 'Header 2': 'What’s the difference between this and RayGraphAdapter?¶'}),\n", " Document(page_content='See https://docs.ray.io/en/latest/workflows/basics.html for the source of the following: \\nFunctions should be idempotent. \\nThe workflow ID is what Ray uses to try to resume/restart if run a second time.', metadata={'Header 1': 'h_ray.RayWorkflowGraphAdapter¶', 'Header 2': 'Tips¶'}),\n", @@ -90112,7 +90112,7 @@ " 'using the Pandas '\n", " 'API on Spark

    \\n'\n", - " '

    Use pip install sf-hamilton[spark] to get '\n", + " '

    Use pip install apache-hamilton[spark] to get '\n", " 'the dependencies required to run this.

    \\n'\n", " '

    Currently, this class assumes you’re running SPARK 3.2+. '\n", " 'You’d generally use this if you have an existing spark '\n", @@ -90615,7 +90615,7 @@ " 'chunks': [Document(page_content='This is an experimental GraphAdapter; there is a possibility of their API changing. That said, the code is stable, and you should feel comfortable giving the code for a spin - let us know how it goes, and what the rough edges are if you find any. We’d love', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", " Document(page_content='are if you find any. We’d love feedback if you are using these to know how to improve them or graduate them.', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", " Document(page_content='Class representing what’s required to make Hamilton run on Spark with Koalas, i.e. Pandas on Spark. \\nThis walks the graph and translates it to run onto Apache Spark using the Pandas API on Spark', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", - " Document(page_content='Use pip install sf-hamilton[spark] to get the dependencies required to run this.', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", + " Document(page_content='Use pip install apache-hamilton[spark] to get the dependencies required to run this.', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", " Document(page_content='Currently, this class assumes you’re running SPARK 3.2+. You’d generally use this if you have an existing spark cluster running in your workplace, and you want to scale to very large data set sizes.', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", " Document(page_content='Some tips on koalas (before it was merged into spark 3.2): \\nhttps://databricks.com/blog/2020/03/31/10-minutes-from-pandas-to-koalas-on-apache-spark.html \\nhttps://spark.apache.org/docs/latest/api/python/user_guide/pandas_on_spark/index.html', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", " Document(page_content='Spark is a more heavyweight choice to scale computation for Hamilton graphs creating a Pandas Dataframe.', metadata={'Header 1': 'h_spark.SparkKoalasGraphAdapter¶'}),\n", diff --git a/examples/LLM_Workflows/scraping_and_chunking/requirements.txt b/examples/LLM_Workflows/scraping_and_chunking/requirements.txt index 2bbeba39e..5813875d4 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/requirements.txt +++ b/examples/LLM_Workflows/scraping_and_chunking/requirements.txt @@ -1,6 +1,6 @@ +apache-hamilton[visualization] langchain langchain-core -sf-hamilton[visualization] # optionally install Ray, or Dask, or both -# sf-hamilton[ray] -# sf-hamilton[dask] +# apache-hamilton[ray] +# apache-hamilton[dask] diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb b/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb index 35dde01c5..6f48a4252 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb +++ b/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/requirements.txt b/examples/LLM_Workflows/scraping_and_chunking/spark/requirements.txt index 42898b943..ea6e7e17b 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/spark/requirements.txt +++ b/examples/LLM_Workflows/scraping_and_chunking/spark/requirements.txt @@ -1,7 +1,7 @@ +apache-hamilton[visualization] langchain langchain-core pyspark -sf-hamilton[visualization] # optionally install Ray, or Dask, or both -# sf-hamilton[ray] -# sf-hamilton[dask] +# apache-hamilton[ray] +# apache-hamilton[dask] diff --git a/examples/README.md b/examples/README.md index a5771283c..0f9361343 100644 --- a/examples/README.md +++ b/examples/README.md @@ -45,8 +45,8 @@ cd hamilton/examples docker build --tag hamilton-example . ``` Docker build takes around `6m16.298s` depending on the system configuration and network. -Alternatively, you can pull the container image from https://hub.docker.com/r/skrawcz/sf-hamilton. -`docker pull skrawcz/sf-hamilton`. +Alternatively, you can pull the container image from https://hub.docker.com/r/skrawcz/apache-hamilton. +`docker pull skrawcz/apache-hamilton`. 3. Starting the container. If you built it yourself: @@ -55,7 +55,7 @@ docker run -it --rm --name hamilton-example hamilton-example ``` If you pulled it from dockerhub: ```bash -docker run -it --rm --name hamilton-example skrawcz/sf-hamilton +docker run -it --rm --name hamilton-example skrawcz/apache-hamilton ``` This will start the container and put you into a bash prompt. diff --git a/examples/airflow/requirements.txt b/examples/airflow/requirements.txt index a5c42edaa..5c0d552eb 100644 --- a/examples/airflow/requirements.txt +++ b/examples/airflow/requirements.txt @@ -1,6 +1,6 @@ +apache-hamilton +apache-hamilton[visualization] numpy pandas pyarrow scikit-learn -sf-hamilton -sf-hamilton[visualization] diff --git a/examples/async/README.md b/examples/async/README.md index 8d904132e..ecafaa4ee 100644 --- a/examples/async/README.md +++ b/examples/async/README.md @@ -66,7 +66,7 @@ curl -X 'POST' \ Recall, to get the server running, you'll have to run the following: ```bash -pip install sf-hamilton[ui] +pip install apache-hamilton[ui] hamilton ui ``` diff --git a/examples/async/requirements.txt b/examples/async/requirements.txt index ef67cc24b..cfd642e02 100644 --- a/examples/async/requirements.txt +++ b/examples/async/requirements.txt @@ -1,4 +1,4 @@ aiohttp +apache-hamilton fastapi -sf-hamilton uvicorn diff --git a/examples/aws/glue/README.md b/examples/aws/glue/README.md index 7e7c6f070..dcb676d87 100644 --- a/examples/aws/glue/README.md +++ b/examples/aws/glue/README.md @@ -114,7 +114,7 @@ First things first, AWS Glue jobs run a single python script, but you can includ --role GlueProcessorRole \ --command '{"Name" : "pythonshell", "PythonVersion": "3.9", "ScriptLocation" : "s3:///processing.py"}' \ --max-capacity 0.0625 \ - --default-arguments '{"--extra-py-files" : "s3:///hamilton_functions-0.1-py3-none-any.whl", "--additional-python-modules" : "sf-hamilton"}' + --default-arguments '{"--extra-py-files" : "s3:///hamilton_functions-0.1-py3-none-any.whl", "--additional-python-modules" : "apache-hamilton"}' ``` - **Run the job:** diff --git a/examples/aws/lambda/requirements.txt b/examples/aws/lambda/requirements.txt index 9ab31ce9b..dfa5a1984 100644 --- a/examples/aws/lambda/requirements.txt +++ b/examples/aws/lambda/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton pandas -sf-hamilton diff --git a/examples/aws/sagemaker/container/requirements.txt b/examples/aws/sagemaker/container/requirements.txt index 78380e35c..26dad87c6 100644 --- a/examples/aws/sagemaker/container/requirements.txt +++ b/examples/aws/sagemaker/container/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton[visualization] pandas -sf-hamilton[visualization] diff --git a/examples/aws/sagemaker/notebook.ipynb b/examples/aws/sagemaker/notebook.ipynb index f89281d9b..fa73ab512 100644 --- a/examples/aws/sagemaker/notebook.ipynb +++ b/examples/aws/sagemaker/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/caching/in_memory_tutorial.ipynb b/examples/caching/in_memory_tutorial.ipynb index d79d0abcd..7a419c86f 100644 --- a/examples/caching/in_memory_tutorial.ipynb +++ b/examples/caching/in_memory_tutorial.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/caching/materializer_tutorial.ipynb b/examples/caching/materializer_tutorial.ipynb index 8235db1d9..953d0fc7e 100644 --- a/examples/caching/materializer_tutorial.ipynb +++ b/examples/caching/materializer_tutorial.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/caching/requirements.txt b/examples/caching/requirements.txt index e90acb2ce..6879c43b6 100644 --- a/examples/caching/requirements.txt +++ b/examples/caching/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton[visualization] ipywidgets pandas -sf-hamilton[visualization] diff --git a/examples/caching/tutorial.ipynb b/examples/caching/tutorial.ipynb index a4eaed8d2..36576d661 100644 --- a/examples/caching/tutorial.ipynb +++ b/examples/caching/tutorial.ipynb @@ -15,7 +15,7 @@ "source": [ "# Execute this cell to install dependencies\n", "## ignore_ci\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/caching_nodes/caching.ipynb b/examples/caching_nodes/caching.ipynb index 64b2d66a7..68ee49171 100644 --- a/examples/caching_nodes/caching.ipynb +++ b/examples/caching_nodes/caching.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb b/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb index cb48f4f08..4b8ce751e 100644 --- a/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb +++ b/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/caching_nodes/caching_graph_adapter/requirements.txt b/examples/caching_nodes/caching_graph_adapter/requirements.txt index 78be7b39a..386c85ab8 100644 --- a/examples/caching_nodes/caching_graph_adapter/requirements.txt +++ b/examples/caching_nodes/caching_graph_adapter/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton >= 1.24.0 pandas pyarrow -sf-hamilton >= 1.24.0 diff --git a/examples/caching_nodes/diskcache_adapter/notebook.ipynb b/examples/caching_nodes/diskcache_adapter/notebook.ipynb index 8d517b7c4..ad97c2f7b 100644 --- a/examples/caching_nodes/diskcache_adapter/notebook.ipynb +++ b/examples/caching_nodes/diskcache_adapter/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/caching_nodes/diskcache_adapter/requirements.txt b/examples/caching_nodes/diskcache_adapter/requirements.txt index eb3adccf6..c21e17289 100644 --- a/examples/caching_nodes/diskcache_adapter/requirements.txt +++ b/examples/caching_nodes/diskcache_adapter/requirements.txt @@ -1 +1 @@ -sf-hamilton[diskcache] +apache-hamilton[diskcache] diff --git a/examples/cli/README.md b/examples/cli/README.md index 6f20110ef..e11aa7eb3 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -24,7 +24,7 @@ The Apache Hamilton CLI allows to build `Driver` objects from the command line. # Installation Install dependencies with (it only needs `typer`) -```pip install sf-hamilton[cli]``` +```pip install apache-hamilton[cli]``` Test the installation with diff --git a/examples/cli/requirements.txt b/examples/cli/requirements.txt index 443ea39c6..23c3cea4f 100644 --- a/examples/cli/requirements.txt +++ b/examples/cli/requirements.txt @@ -1 +1 @@ -sf-hamilton[cli] +apache-hamilton[cli] diff --git a/examples/contrib/README.md b/examples/contrib/README.md index ad4737190..f51e35d4c 100644 --- a/examples/contrib/README.md +++ b/examples/contrib/README.md @@ -37,7 +37,7 @@ Or run it in Google Colab: There are 3 main ways to use community dataflows: static installation, dynamic installation, and local copy (see [documentation](https://github.com/apache/hamilton/tree/main/contrib)). We present each of them in this example: ## 1. Static installation -The script `run.py` uses the direct import `from hamilton.contrib.user.zilto import xgboost_optuna`. It's as simple as that! (but first `pip install sf-hamilton-contrib --upgrade`) +The script `run.py` uses the direct import `from hamilton.contrib.user.zilto import xgboost_optuna`. It's as simple as that! (but first `pip install apache-hamilton-contrib --upgrade`) ## 2. Dynamic installation The first part of the notebook `notebook.ipynb` imports the same dataflow via `xgboost_optuna = hamilton.dataflows.import_module("xgboost_optuna", "zilto")`. This will download and cache the module in your local directory `{USER_PATH}/.hamilton`. diff --git a/examples/contrib/notebook.ipynb b/examples/contrib/notebook.ipynb index 08ee14005..da22b8330 100644 --- a/examples/contrib/notebook.ipynb +++ b/examples/contrib/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -40,7 +40,7 @@ }, "source": [ "# 1. Install from PyPI\n", - "Install the package from PyPI with `!pip install sf-hamilton-contrib`.\n", + "Install the package from PyPI with `!pip install apache-hamilton-contrib`.\n", "Then see run.py for details. " ] }, @@ -50,7 +50,7 @@ "metadata": {}, "outputs": [], "source": [ - "# !pip install sf-hamilton-contrib" + "# !pip install apache-hamilton-contrib" ] }, { @@ -58,7 +58,7 @@ "metadata": {}, "source": [ "# 2. Dynamic installation\n", - "Install the package dynamically with without installing/upgrading sf-hamilton-contrib." + "Install the package dynamically with without installing/upgrading apache-hamilton-contrib." ] }, { diff --git a/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb b/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb index 0ececf583..7a4b699ec 100644 --- a/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb +++ b/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/contrib/requirements.txt b/examples/contrib/requirements.txt index 9463abc03..c0765f649 100644 --- a/examples/contrib/requirements.txt +++ b/examples/contrib/requirements.txt @@ -1,7 +1,7 @@ +apache-hamilton-contrib +apache-hamilton[visualization] numpy optuna pandas -sf-hamilton-contrib -sf-hamilton[visualization] sklearn xgboost diff --git a/examples/dagster/hamilton_code/notebook.ipynb b/examples/dagster/hamilton_code/notebook.ipynb index e9980979a..560d4052b 100644 --- a/examples/dagster/hamilton_code/notebook.ipynb +++ b/examples/dagster/hamilton_code/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/dagster/requirements.txt b/examples/dagster/requirements.txt index a234fbb85..ff423ecdd 100644 --- a/examples/dagster/requirements.txt +++ b/examples/dagster/requirements.txt @@ -1,3 +1,4 @@ +apache-hamilton[visualization] dagster dagster-webserver dagster_duckdb @@ -6,4 +7,3 @@ Faker==18.4.0 matplotlib pandas requests -sf-hamilton[visualization] diff --git a/examples/dask/community_demo/demo_day_notebook.ipynb b/examples/dask/community_demo/demo_day_notebook.ipynb index 87d0371fa..b22543c9f 100644 --- a/examples/dask/community_demo/demo_day_notebook.ipynb +++ b/examples/dask/community_demo/demo_day_notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[dask,visualization]" + "%pip install apache-hamilton[dask,visualization]" ] }, { @@ -250,7 +250,7 @@ ], "source": [ "# Cell 4 - we can visualize the entire DAG\n", - "# To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + "# To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", "\n", "# visualize all possible nodes\n", "dr.display_all_functions() # no extra args needed for jupyter" diff --git a/examples/dask/community_demo/requirements.txt b/examples/dask/community_demo/requirements.txt index 58c0e8f10..923ae646e 100644 --- a/examples/dask/community_demo/requirements.txt +++ b/examples/dask/community_demo/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton[dask,visualization] jupyter pandas -sf-hamilton[dask,visualization] diff --git a/examples/dask/hello_world/README.md b/examples/dask/hello_world/README.md index 83006ba67..7c3c64d0a 100644 --- a/examples/dask/hello_world/README.md +++ b/examples/dask/hello_world/README.md @@ -26,7 +26,7 @@ in a distributed setting via dask. Note: please read this [dask best practices post](https://docs.dask.org/en/stable/dataframe-best-practices.html); don't scale if you don't need to. -`pip install sf-hamilton[dask-complete]` or `pip install sf-hamilton dask[complete]` to for the right dependencies to run this example. +`pip install apache-hamilton[dask-complete]` or `pip install apache-hamilton dask[complete]` to for the right dependencies to run this example. File organization: diff --git a/examples/dask/hello_world/notebook.ipynb b/examples/dask/hello_world/notebook.ipynb index eba1489e9..f07c36d11 100644 --- a/examples/dask/hello_world/notebook.ipynb +++ b/examples/dask/hello_world/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -501,7 +501,7 @@ "dask_df = dr.execute(output_columns) # it's dask dataframe -- it hasn't been evaluated yet.\n", "df = dask_df.compute()\n", "\n", - "# To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + "# To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", "dr.visualize_execution(output_columns)" ] }, diff --git a/examples/dask/hello_world/run.py b/examples/dask/hello_world/run.py index 4a8853230..06da36214 100644 --- a/examples/dask/hello_world/run.py +++ b/examples/dask/hello_world/run.py @@ -53,7 +53,7 @@ use_delayed=False, compute_at_end=False, ) - # will output Dask's execution graph -- requires sf-hamilton[visualization] to be installed. + # will output Dask's execution graph -- requires apache-hamilton[visualization] to be installed. initial_config_and_data = { "spend_location": "some file path", @@ -75,7 +75,7 @@ ] dask_df = dr.execute(output_columns) # it's dask dataframe -- it hasn't been evaluated yet. df = dask_df.compute() - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './hello_world_dask', {"format": "png"}) dr.display_all_functions("./my_full_dag.dot") logger.info(df.to_string()) diff --git a/examples/dask/hello_world/run_with_delayed.py b/examples/dask/hello_world/run_with_delayed.py index 196d080d4..9e456db3a 100644 --- a/examples/dask/hello_world/run_with_delayed.py +++ b/examples/dask/hello_world/run_with_delayed.py @@ -50,7 +50,7 @@ use_delayed=True, compute_at_end=True, ) - # will output Dask's execution graph run_with_delayed.png -- requires sf-hamilton[visualization] to be installed. + # will output Dask's execution graph run_with_delayed.png -- requires apache-hamilton[visualization] to be installed. initial_config_and_data = { "signups": pd.Series([1, 10, 50, 100, 200, 400]), @@ -68,7 +68,7 @@ ] pandas_df = dr.execute(output_columns) logger.info(pandas_df.to_string()) - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './hello_world_dask', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') client.shutdown() diff --git a/examples/dask/hello_world/run_with_delayed_and_dask_objects.py b/examples/dask/hello_world/run_with_delayed_and_dask_objects.py index c3b3bcebc..c44fc0354 100644 --- a/examples/dask/hello_world/run_with_delayed_and_dask_objects.py +++ b/examples/dask/hello_world/run_with_delayed_and_dask_objects.py @@ -51,7 +51,7 @@ use_delayed=True, compute_at_end=True, ) - # will output Dask's execution graph -- requires sf-hamilton[visualization] to be installed. + # will output Dask's execution graph -- requires apache-hamilton[visualization] to be installed. initial_config_and_data = { "spend_location": "some file path", "spend_partitions": 2, @@ -73,7 +73,7 @@ dask_df = dr.execute(output_columns) # it's dask dataframe -- it hasn't been evaluated yet. logger.info(dask_df.to_string()) # this should be empty df = dask_df.compute() - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './hello_world_dask', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') logger.info(df.to_string()) diff --git a/examples/dask/requirements.txt b/examples/dask/requirements.txt index df769dbe3..385469cf9 100644 --- a/examples/dask/requirements.txt +++ b/examples/dask/requirements.txt @@ -1 +1 @@ -sf-hamilton[dask] +apache-hamilton[dask] diff --git a/examples/data_loaders/data_loaders.ipynb b/examples/data_loaders/data_loaders.ipynb index f1e5333b2..51b382430 100644 --- a/examples/data_loaders/data_loaders.ipynb +++ b/examples/data_loaders/data_loaders.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -62,7 +62,7 @@ "metadata": {}, "outputs": [], "source": [ - "# %pip install sf-hamilton[visualization] pandas duckdb==0.5.0" + "# %pip install apache-hamilton[visualization] pandas duckdb==0.5.0" ] }, { diff --git a/examples/data_quality/pandera/requirements-dask.txt b/examples/data_quality/pandera/requirements-dask.txt index 39c414f88..ae32c36f4 100644 --- a/examples/data_quality/pandera/requirements-dask.txt +++ b/examples/data_quality/pandera/requirements-dask.txt @@ -1,2 +1,2 @@ -sf-hamilton[pandera, dask] -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton[pandera, dask] +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/pandera/requirements-ray.txt b/examples/data_quality/pandera/requirements-ray.txt index 118e924ff..1cc7a7512 100644 --- a/examples/data_quality/pandera/requirements-ray.txt +++ b/examples/data_quality/pandera/requirements-ray.txt @@ -1,2 +1,2 @@ -sf-hamilton[pandera, ray] -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton[pandera, ray] +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/pandera/requirements-spark.txt b/examples/data_quality/pandera/requirements-spark.txt index e2b47ada1..d8666ef57 100644 --- a/examples/data_quality/pandera/requirements-spark.txt +++ b/examples/data_quality/pandera/requirements-spark.txt @@ -1,2 +1,2 @@ -sf-hamilton[pandera, spark] -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton[pandera, spark] +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/pandera/requirements.txt b/examples/data_quality/pandera/requirements.txt index dcad069a3..040368326 100644 --- a/examples/data_quality/pandera/requirements.txt +++ b/examples/data_quality/pandera/requirements.txt @@ -1,2 +1,2 @@ -sf-hamilton[pandera] -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton[pandera] +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/pandera/run.py b/examples/data_quality/pandera/run.py index 1e818ff0b..ecc07eb30 100644 --- a/examples/data_quality/pandera/run.py +++ b/examples/data_quality/pandera/run.py @@ -60,7 +60,7 @@ "seasons_4", "absenteeism_time_in_hours", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './run', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/data_quality/pandera/run_dask.py b/examples/data_quality/pandera/run_dask.py index bb5c2e038..888fe7cf3 100644 --- a/examples/data_quality/pandera/run_dask.py +++ b/examples/data_quality/pandera/run_dask.py @@ -77,7 +77,7 @@ "seasons_4", "absenteeism_time_in_hours", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './run_dask', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/data_quality/pandera/run_ray.py b/examples/data_quality/pandera/run_ray.py index 5e6524e8a..281454d19 100644 --- a/examples/data_quality/pandera/run_ray.py +++ b/examples/data_quality/pandera/run_ray.py @@ -73,7 +73,7 @@ "seasons_4", "absenteeism_time_in_hours", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './my_dag.dot', {}, graphviz_kwargs=dict(graph_attr={'ratio': "1"})) # dr.visualize_execution(output_columns, './run_ray', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/data_quality/pandera/run_spark.py b/examples/data_quality/pandera/run_spark.py index cc73fc02c..2cb190194 100644 --- a/examples/data_quality/pandera/run_spark.py +++ b/examples/data_quality/pandera/run_spark.py @@ -81,7 +81,7 @@ "absenteeism_time_in_hours", "index_col", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './run_spark', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/data_quality/simple/requirements-dask.txt b/examples/data_quality/simple/requirements-dask.txt index 46ea8084c..5cb747906 100644 --- a/examples/data_quality/simple/requirements-dask.txt +++ b/examples/data_quality/simple/requirements-dask.txt @@ -1,2 +1,2 @@ -sf-hamilton[dask] -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton[dask] +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/simple/requirements-ray.txt b/examples/data_quality/simple/requirements-ray.txt index e2a170638..2df5d1847 100644 --- a/examples/data_quality/simple/requirements-ray.txt +++ b/examples/data_quality/simple/requirements-ray.txt @@ -1,2 +1,2 @@ -sf-hamilton[ray] -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton[ray] +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/simple/requirements-spark.txt b/examples/data_quality/simple/requirements-spark.txt index b1187aacd..078dcca1d 100644 --- a/examples/data_quality/simple/requirements-spark.txt +++ b/examples/data_quality/simple/requirements-spark.txt @@ -1,2 +1,2 @@ -sf-hamilton[spark] -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton[spark] +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/simple/requirements.txt b/examples/data_quality/simple/requirements.txt index 1e2d99f7e..f00d1c756 100644 --- a/examples/data_quality/simple/requirements.txt +++ b/examples/data_quality/simple/requirements.txt @@ -1,2 +1,2 @@ -sf-hamilton -# sf-hamilton[visualization] # if visualization is desired +apache-hamilton +# apache-hamilton[visualization] # if visualization is desired diff --git a/examples/data_quality/simple/run.py b/examples/data_quality/simple/run.py index 1e818ff0b..ecc07eb30 100644 --- a/examples/data_quality/simple/run.py +++ b/examples/data_quality/simple/run.py @@ -60,7 +60,7 @@ "seasons_4", "absenteeism_time_in_hours", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './run', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/data_quality/simple/run_dask.py b/examples/data_quality/simple/run_dask.py index 3febec7cf..197b72b26 100644 --- a/examples/data_quality/simple/run_dask.py +++ b/examples/data_quality/simple/run_dask.py @@ -77,7 +77,7 @@ "seasons_4", "absenteeism_time_in_hours", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './run_dask', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/data_quality/simple/run_ray.py b/examples/data_quality/simple/run_ray.py index 7c3739ed6..307b7e782 100644 --- a/examples/data_quality/simple/run_ray.py +++ b/examples/data_quality/simple/run_ray.py @@ -73,7 +73,7 @@ "seasons_4", "absenteeism_time_in_hours", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './run_ray', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/data_quality/simple/run_spark.py b/examples/data_quality/simple/run_spark.py index 5003f5ef9..d36da8ffa 100644 --- a/examples/data_quality/simple/run_spark.py +++ b/examples/data_quality/simple/run_spark.py @@ -81,7 +81,7 @@ "absenteeism_time_in_hours", "index_col", ] - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './run_spark', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/dbt/models/train_and_infer.py b/examples/dbt/models/train_and_infer.py index 45bce6f19..637a5d5fb 100644 --- a/examples/dbt/models/train_and_infer.py +++ b/examples/dbt/models/train_and_infer.py @@ -49,7 +49,7 @@ def model(dbt, session): results = titanic_dag.execute( final_vars=["model_predict"], inputs={"raw_passengers_df": raw_passengers_df} ) - # pip install "sf-hamilton[visualization]" to get this to work + # pip install "apache-hamilton[visualization]" to get this to work # titanic_dag.visualize_execution(["model_predict"], './titanic_dbt', {"format": "png"}, # inputs={"raw_passengers_df": raw_passengers_df}) # Take the "predictions" result, which is an np array diff --git a/examples/dbt/requirements.txt b/examples/dbt/requirements.txt index d85ace805..8fc09f914 100644 --- a/examples/dbt/requirements.txt +++ b/examples/dbt/requirements.txt @@ -1,3 +1,4 @@ +apache-hamilton dbt-core dbt-duckdb dbt-fal @@ -5,4 +6,3 @@ dbt-python duckdb==0.6.1 # required to read saved database.duckdb file pandera scikit-learn -sf-hamilton diff --git a/examples/decoupling_io/requirements.txt b/examples/decoupling_io/requirements.txt index e5794b624..30d78b424 100644 --- a/examples/decoupling_io/requirements.txt +++ b/examples/decoupling_io/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton click scikit-learn -sf-hamilton diff --git a/examples/dlt/dlt_plugin.ipynb b/examples/dlt/dlt_plugin.ipynb index b2a73decd..bf7645f4e 100644 --- a/examples/dlt/dlt_plugin.ipynb +++ b/examples/dlt/dlt_plugin.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] dlt" + "%pip install apache-hamilton[visualization] dlt" ] }, { diff --git a/examples/dlt/notebook.ipynb b/examples/dlt/notebook.ipynb index 7f141bce3..60cfdbd3a 100644 --- a/examples/dlt/notebook.ipynb +++ b/examples/dlt/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] dlt[duckdb]>=0.3.12 ibis-framework[duckdb] openai pandas polars" + "%pip install apache-hamilton[visualization] dlt[duckdb]>=0.3.12 ibis-framework[duckdb] openai pandas polars" ] }, { diff --git a/examples/dlt/requirements.txt b/examples/dlt/requirements.txt index ea9a32ac5..c7eb3a30e 100644 --- a/examples/dlt/requirements.txt +++ b/examples/dlt/requirements.txt @@ -1,7 +1,7 @@ +apache-hamilton[visualization] dlt[duckdb]>=0.3.12 ibis-framework[duckdb] openai pandas polars -sf-hamilton[visualization] streamlit diff --git a/examples/due_date_probabilities/notebook.ipynb b/examples/due_date_probabilities/notebook.ipynb index ed1d5439e..7495b54c6 100644 --- a/examples/due_date_probabilities/notebook.ipynb +++ b/examples/due_date_probabilities/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] pandas scipy seaborn matplotlib" + "%pip install apache-hamilton[visualization] pandas scipy seaborn matplotlib" ] }, { diff --git a/examples/experiment_management/README.md b/examples/experiment_management/README.md index 2e6eea4f2..960799a99 100644 --- a/examples/experiment_management/README.md +++ b/examples/experiment_management/README.md @@ -31,7 +31,7 @@ Add a hook to your Apache Hamilton Driver to log runs and visualize artifacts an - 📡 Launch a local server to view and explore your experiments ## Installation -Use `pip install sf-hamilton[experiments]` to install both the hook and the server with their dependencies +Use `pip install apache-hamilton[experiments]` to install both the hook and the server with their dependencies ## How to use the ExperimentTracker hook The `ExperimentTracker` hook can be added to your Apache Hamilton Driver definition to automatically log metadata about the run and materialized results. diff --git a/examples/experiment_management/notebook.ipynb b/examples/experiment_management/notebook.ipynb index e57ef7045..82e335c12 100644 --- a/examples/experiment_management/notebook.ipynb +++ b/examples/experiment_management/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/experiment_management/requirements.txt b/examples/experiment_management/requirements.txt index 0cd7f3874..636d4f2ef 100644 --- a/examples/experiment_management/requirements.txt +++ b/examples/experiment_management/requirements.txt @@ -1,7 +1,7 @@ +apache-hamilton[experiments,visualization] fastparquet matplotlib numpy pandas pyarrow scikit-learn -sf-hamilton[experiments,visualization] diff --git a/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb b/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb index 98f9d55a2..660a1ae1f 100644 --- a/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb +++ b/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/feast/requirements.txt b/examples/feast/requirements.txt index e9907c8c3..f3e814354 100644 --- a/examples/feast/requirements.txt +++ b/examples/feast/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton[visualization] feast jupyter pandas -sf-hamilton[visualization] diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/requirements.txt b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/requirements.txt index e5ad16c12..4f0f7b0e5 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/requirements.txt +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton fastapi pandas pandera -sf-hamilton uvicorn diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/requirements.txt b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/requirements.txt index 4857ae7e8..31301c400 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/requirements.txt +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/requirements.txt @@ -1,6 +1,6 @@ aiohttp +apache-hamilton fastapi pandas pandera -sf-hamilton uvicorn diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py index 482df044f..491c3b09a 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py @@ -49,7 +49,7 @@ model, adapter=base.DefaultAdapter(), ) -# pip install "sf-hamilton[visualization]" for this next line to work +# pip install "apache-hamilton[visualization]" for this next line to work # dr.display_all_functions("streaming", {"format": "png"}) diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/requirements.txt b/examples/feature_engineering/write_once_run_everywhere_blog_post/requirements.txt index 379c24484..3570efdf4 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/requirements.txt +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton bytewax fastapi -sf-hamilton uvicorn diff --git a/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb b/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb index 7420767fa..ab8c4f916 100644 --- a/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] pandas scikit-learn numpy" + "%pip install apache-hamilton[visualization] pandas scikit-learn numpy" ] }, { diff --git a/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb b/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb index 01f1a86f0..9646df0fb 100644 --- a/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] pandas scikit-learn numpy" + "%pip install apache-hamilton[visualization] pandas scikit-learn numpy" ] }, { diff --git a/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb b/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb index 122e7e42e..88256dfc4 100644 --- a/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] pandas scikit-learn numpy" + "%pip install apache-hamilton[visualization] pandas scikit-learn numpy" ] }, { diff --git a/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb b/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb index 93be0c73a..028b8099d 100644 --- a/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] pandas scikit-learn numpy" + "%pip install apache-hamilton[visualization] pandas scikit-learn numpy" ] }, { diff --git a/examples/hamilton-tutorials/mpg-translation/requirements.txt b/examples/hamilton-tutorials/mpg-translation/requirements.txt index 6c5027af9..1f0ac5a1c 100644 --- a/examples/hamilton-tutorials/mpg-translation/requirements.txt +++ b/examples/hamilton-tutorials/mpg-translation/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton[visualization,sdk,ui] numpy pandas scikit-learn -sf-hamilton[visualization,sdk,ui] diff --git a/examples/hamilton_ui/README.md b/examples/hamilton_ui/README.md index ae6e7fbf6..a02d83b2f 100644 --- a/examples/hamilton_ui/README.md +++ b/examples/hamilton_ui/README.md @@ -37,7 +37,7 @@ First, you need to have the Apache Hamilton UI running. You can either `pip inst Install the Python dependencies: ```bash -pip install "sf-hamilton[ui,sdk]" +pip install "apache-hamilton[ui,sdk]" ``` then launch the Apache Hamilton UI server: ```bash diff --git a/examples/hamilton_ui/notebook.ipynb b/examples/hamilton_ui/notebook.ipynb index dd0f897f8..4a732de42 100644 --- a/examples/hamilton_ui/notebook.ipynb +++ b/examples/hamilton_ui/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/hamilton_ui/requirements.txt b/examples/hamilton_ui/requirements.txt index 5e71c3359..a942e9949 100644 --- a/examples/hamilton_ui/requirements.txt +++ b/examples/hamilton_ui/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton[sdk] click pandas pyarrow scikit-learn -sf-hamilton[sdk] diff --git a/examples/hello_world/my_notebook.ipynb b/examples/hello_world/my_notebook.ipynb index 7ba00536f..df77d3594 100644 --- a/examples/hello_world/my_notebook.ipynb +++ b/examples/hello_world/my_notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -597,7 +597,7 @@ ], "source": [ "# Cell 5 - visualize execution\n", - "# To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + "# To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", "\n", "# visualize all possible functions\n", "dr.display_all_functions() # no args needed for jupyter" diff --git a/examples/hello_world/my_script.py b/examples/hello_world/my_script.py index 8822fc73d..e38858610 100644 --- a/examples/hello_world/my_script.py +++ b/examples/hello_world/my_script.py @@ -52,6 +52,6 @@ df = dr.execute(output_columns, inputs=initial_columns) print(df.to_string()) -# To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work +# To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # Also look up visualize_path_between and visualize_execution dr.display_all_functions("./my_dag.png") diff --git a/examples/hello_world/requirements.txt b/examples/hello_world/requirements.txt index 9ab31ce9b..dfa5a1984 100644 --- a/examples/hello_world/requirements.txt +++ b/examples/hello_world/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton pandas -sf-hamilton diff --git a/examples/ibis/feature_engineering/requirements.txt b/examples/ibis/feature_engineering/requirements.txt index 54412d8cb..e5eef1117 100644 --- a/examples/ibis/feature_engineering/requirements.txt +++ b/examples/ibis/feature_engineering/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton[visualization] ibis-framework[duckdb,examples] -sf-hamilton[visualization] tdqm diff --git a/examples/ibis/jaffle_shop/requirements.txt b/examples/ibis/jaffle_shop/requirements.txt index d81448183..b016faee5 100644 --- a/examples/ibis/jaffle_shop/requirements.txt +++ b/examples/ibis/jaffle_shop/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton[visualization] ibis-framework[duckdb] -sf-hamilton[visualization] diff --git a/examples/ibisml/requirements.txt b/examples/ibisml/requirements.txt index 532313658..1256878ff 100644 --- a/examples/ibisml/requirements.txt +++ b/examples/ibisml/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton[visualization] ibis-framework[duckdb,examples] ibisml scikit-learn -sf-hamilton[visualization] tdqm diff --git a/examples/jupyter_notebook_magic/example.ipynb b/examples/jupyter_notebook_magic/example.ipynb index da786283b..1ab2f3ac8 100644 --- a/examples/jupyter_notebook_magic/example.ipynb +++ b/examples/jupyter_notebook_magic/example.ipynb @@ -25,7 +25,7 @@ ], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/kedro/hamilton-code/notebooks/interactive.ipynb b/examples/kedro/hamilton-code/notebooks/interactive.ipynb index 14e291624..52245c2dc 100644 --- a/examples/kedro/hamilton-code/notebooks/interactive.ipynb +++ b/examples/kedro/hamilton-code/notebooks/interactive.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/kedro/hamilton-code/requirements.txt b/examples/kedro/hamilton-code/requirements.txt index cc3d4c951..ec1fda4d4 100644 --- a/examples/kedro/hamilton-code/requirements.txt +++ b/examples/kedro/hamilton-code/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton[visualization] pytest~=7.2 scikit-learn~=1.0 -sf-hamilton[visualization] diff --git a/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb b/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb index 39bb6eefa..47d15ca46 100644 --- a/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb +++ b/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/lineage/lineage_snippets.ipynb b/examples/lineage/lineage_snippets.ipynb index 8f98289e3..a1c65ee2b 100644 --- a/examples/lineage/lineage_snippets.ipynb +++ b/examples/lineage/lineage_snippets.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/lineage/requirements.txt b/examples/lineage/requirements.txt index 3aefcd492..7c16c2d8b 100644 --- a/examples/lineage/requirements.txt +++ b/examples/lineage/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton gitpython numpy pandas scikit-learn -sf-hamilton diff --git a/examples/materialization/datasaver_dataloader_example/notebook.ipynb b/examples/materialization/datasaver_dataloader_example/notebook.ipynb index 059a5da62..a6f38abb7 100644 --- a/examples/materialization/datasaver_dataloader_example/notebook.ipynb +++ b/examples/materialization/datasaver_dataloader_example/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization,sdk,ui] scikit-learn pandas" + "%pip install apache-hamilton[visualization,sdk,ui] scikit-learn pandas" ] }, { diff --git a/examples/materialization/datasaver_dataloader_example/requirements.txt b/examples/materialization/datasaver_dataloader_example/requirements.txt index 608dc1d9b..603ad0dc8 100644 --- a/examples/materialization/datasaver_dataloader_example/requirements.txt +++ b/examples/materialization/datasaver_dataloader_example/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton[visualization, sdk, ui] pandas scikit-learn -sf-hamilton[visualization, sdk, ui] diff --git a/examples/materialization/notebook.ipynb b/examples/materialization/notebook.ipynb index cc8b8b2fb..560636fa0 100644 --- a/examples/materialization/notebook.ipynb +++ b/examples/materialization/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/materialization/requirements.txt b/examples/materialization/requirements.txt index 3f69ad5c2..559a0b7c0 100644 --- a/examples/materialization/requirements.txt +++ b/examples/materialization/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton scikit-learn -sf-hamilton diff --git a/examples/mlflow/requirements.txt b/examples/mlflow/requirements.txt index 4d5aed0a1..f6ebcbd49 100644 --- a/examples/mlflow/requirements.txt +++ b/examples/mlflow/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton[visualization] mlflow numpy pandas scikit-learn -sf-hamilton[visualization] diff --git a/examples/mlflow/tutorial.ipynb b/examples/mlflow/tutorial.ipynb index 3d5418b72..b94e4340e 100644 --- a/examples/mlflow/tutorial.ipynb +++ b/examples/mlflow/tutorial.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/model_examples/modular_example/notebook.ipynb b/examples/model_examples/modular_example/notebook.ipynb index 1bae32fe4..fb7bafa03 100644 --- a/examples/model_examples/modular_example/notebook.ipynb +++ b/examples/model_examples/modular_example/notebook.ipynb @@ -15,7 +15,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb b/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb index 1d35fdb23..42e978024 100644 --- a/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb +++ b/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -224,7 +224,7 @@ " to my_train_evaluate_logic.\n", "\"\"\"\n", "dr = driver.Driver(dag_config, data_module, my_train_evaluate_logic, adapter=adapter)\n", - "# ensure you have done \"pip install \"sf-hamilton[visualization]\"\" for the following to work:\n", + "# ensure you have done \"pip install \"apache-hamilton[visualization]\"\" for the following to work:\n", "# dr.visualize_execution(['classification_report', 'confusion_matrix', 'fit_clf'],\n", "# f'./model_dag_{_data_set}_{_model_type}.dot', {\"format\": \"png\"})\n", "results = dr.execute([\"classification_report\", \"confusion_matrix\", \"fit_clf\"])\n", diff --git a/examples/model_examples/scikit-learn/requirements.txt b/examples/model_examples/scikit-learn/requirements.txt index 3f69ad5c2..559a0b7c0 100644 --- a/examples/model_examples/scikit-learn/requirements.txt +++ b/examples/model_examples/scikit-learn/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton scikit-learn -sf-hamilton diff --git a/examples/model_examples/scikit-learn/run.py b/examples/model_examples/scikit-learn/run.py index 0928bb5ee..ebf6dd078 100644 --- a/examples/model_examples/scikit-learn/run.py +++ b/examples/model_examples/scikit-learn/run.py @@ -74,7 +74,7 @@ def get_model_config(model_type: str) -> dict: to my_train_evaluate_logic. """ dr = driver.Driver(dag_config, data_module, my_train_evaluate_logic, adapter=adapter) - # ensure you have done "pip install "sf-hamilton[visualization]"" for the following to work: + # ensure you have done "pip install "apache-hamilton[visualization]"" for the following to work: # dr.visualize_execution(['classification_report', 'confusion_matrix', 'fit_clf'], # f'./model_dag_{_data_set}_{_model_type}.dot', {"format": "png"}) results = dr.execute(["classification_report", "confusion_matrix", "fit_clf"]) diff --git a/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb b/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb index 58af3796e..0dafc1d37 100644 --- a/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb +++ b/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/model_examples/time-series/requirements.txt b/examples/model_examples/time-series/requirements.txt index 166426c40..7b3584e0d 100644 --- a/examples/model_examples/time-series/requirements.txt +++ b/examples/model_examples/time-series/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton lightgbm numpy pandas scikit-learn -sf-hamilton diff --git a/examples/mutate/abstract_functionality_blueprint/notebook.ipynb b/examples/mutate/abstract_functionality_blueprint/notebook.ipynb index a1d1ae82d..87c723814 100644 --- a/examples/mutate/abstract_functionality_blueprint/notebook.ipynb +++ b/examples/mutate/abstract_functionality_blueprint/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/narwhals/notebook.ipynb b/examples/narwhals/notebook.ipynb index ccc6e11c2..a31d85e12 100644 --- a/examples/narwhals/notebook.ipynb +++ b/examples/narwhals/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] pandas polars narwhals" + "%pip install apache-hamilton[visualization] pandas polars narwhals" ] }, { diff --git a/examples/narwhals/requirements.txt b/examples/narwhals/requirements.txt index 4c5cbc75a..e88529b6f 100644 --- a/examples/narwhals/requirements.txt +++ b/examples/narwhals/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton[visualization] narwhals pandas polars -sf-hamilton[visualization] diff --git a/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb b/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb index 1364aa38d..f3c90e0a3 100644 --- a/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb +++ b/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -728,7 +728,7 @@ ], "source": [ "# Visualize execution\n", - "# To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + "# To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", "\n", "# visualize all possible functions\n", "dr.display_all_functions() # no args needed for a notebook." diff --git a/examples/numpy/air-quality-analysis/requirements.txt b/examples/numpy/air-quality-analysis/requirements.txt index a8b0c26b4..802477ed2 100644 --- a/examples/numpy/air-quality-analysis/requirements.txt +++ b/examples/numpy/air-quality-analysis/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton numpy scipy -sf-hamilton diff --git a/examples/openlineage/notebook.ipynb b/examples/openlineage/notebook.ipynb index 4913be75b..4d6bb7db6 100644 --- a/examples/openlineage/notebook.ipynb +++ b/examples/openlineage/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/openlineage/requirements.txt b/examples/openlineage/requirements.txt index 88dd9ab74..ae9f2e653 100644 --- a/examples/openlineage/requirements.txt +++ b/examples/openlineage/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton openlineage-python -sf-hamilton diff --git a/examples/pandas/materialization/notebook.ipynb b/examples/pandas/materialization/notebook.ipynb index 1485f2467..6be54895d 100644 --- a/examples/pandas/materialization/notebook.ipynb +++ b/examples/pandas/materialization/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/pandas/materialization/requirements.txt b/examples/pandas/materialization/requirements.txt index 3ba00d435..ed57baeee 100644 --- a/examples/pandas/materialization/requirements.txt +++ b/examples/pandas/materialization/requirements.txt @@ -1,3 +1,3 @@ +apache-hamilton lxml # required for XML pandas -sf-hamilton diff --git a/examples/pandas/split-apply-combine/my_wrapper.py b/examples/pandas/split-apply-combine/my_wrapper.py index 37f0a15cb..2800badfc 100644 --- a/examples/pandas/split-apply-combine/my_wrapper.py +++ b/examples/pandas/split-apply-combine/my_wrapper.py @@ -51,5 +51,5 @@ def calculate( @staticmethod def visualize(output_path="./my_full_dag.png"): - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work driver.display_all_functions(output_path) diff --git a/examples/pandas/split-apply-combine/notebook.ipynb b/examples/pandas/split-apply-combine/notebook.ipynb index ec2bd7966..d0d198236 100644 --- a/examples/pandas/split-apply-combine/notebook.ipynb +++ b/examples/pandas/split-apply-combine/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -560,7 +560,7 @@ "\n", "from hamilton import base, driver, lifecycle\n", "\n", - "## if using the Hamilton UI do `pip install sf-hamilton[ui,sdk]`\n", + "## if using the Hamilton UI do `pip install apache-hamilton[ui,sdk]`\n", "# from hamilton_sdk import adapters\n", "\n", "# tracker = adapters.HamiltonTracker(\n", @@ -601,7 +601,7 @@ "\n", " @staticmethod\n", " def visualize():\n", - " # To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + " # To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", " return driver.display_all_functions()" ] }, diff --git a/examples/pandas/split-apply-combine/requirements.txt b/examples/pandas/split-apply-combine/requirements.txt index 78380e35c..26dad87c6 100644 --- a/examples/pandas/split-apply-combine/requirements.txt +++ b/examples/pandas/split-apply-combine/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton[visualization] pandas -sf-hamilton[visualization] diff --git a/examples/pandas/with_columns/notebook.ipynb b/examples/pandas/with_columns/notebook.ipynb index 8a00133a8..d0c1b35f7 100644 --- a/examples/pandas/with_columns/notebook.ipynb +++ b/examples/pandas/with_columns/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/parallelism/file_processing/notebook.ipynb b/examples/parallelism/file_processing/notebook.ipynb index 00c9feb6e..fcbf1f2b4 100644 --- a/examples/parallelism/file_processing/notebook.ipynb +++ b/examples/parallelism/file_processing/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/parallelism/file_processing/requirements.txt b/examples/parallelism/file_processing/requirements.txt index fe0e26dba..3d233ec29 100644 --- a/examples/parallelism/file_processing/requirements.txt +++ b/examples/parallelism/file_processing/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton>=1.25.0 dask jupyter matplotlib ray -sf-hamilton>=1.25.0 diff --git a/examples/parallelism/lazy_threadpool_execution/notebook.ipynb b/examples/parallelism/lazy_threadpool_execution/notebook.ipynb index 4d9f15d21..838433f9a 100644 --- a/examples/parallelism/lazy_threadpool_execution/notebook.ipynb +++ b/examples/parallelism/lazy_threadpool_execution/notebook.ipynb @@ -18,7 +18,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/parallelism/lazy_threadpool_execution/requirements.txt b/examples/parallelism/lazy_threadpool_execution/requirements.txt index 0bd997a6e..abee97199 100644 --- a/examples/parallelism/lazy_threadpool_execution/requirements.txt +++ b/examples/parallelism/lazy_threadpool_execution/requirements.txt @@ -1,3 +1,3 @@ -sf-hamilton-sdk # optional -sf-hamilton-ui # optional -sf-hamilton[visualization] +apache-hamilton-sdk # optional +apache-hamilton-ui # optional +apache-hamilton[visualization] diff --git a/examples/parallelism/star_counting/notebook.ipynb b/examples/parallelism/star_counting/notebook.ipynb index fc6bacede..2aa6b6a17 100644 --- a/examples/parallelism/star_counting/notebook.ipynb +++ b/examples/parallelism/star_counting/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/people_data_labs/notebook.ipynb b/examples/people_data_labs/notebook.ipynb index 41e7fb449..d10685dc3 100644 --- a/examples/people_data_labs/notebook.ipynb +++ b/examples/people_data_labs/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/people_data_labs/requirements.txt b/examples/people_data_labs/requirements.txt index 929b343e1..c79808c62 100644 --- a/examples/people_data_labs/requirements.txt +++ b/examples/people_data_labs/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton[visualization] boto3 pandas pyarrow -sf-hamilton[visualization] diff --git a/examples/plotly/notebook.ipynb b/examples/plotly/notebook.ipynb index ddc13c6d8..b5354e1bf 100644 --- a/examples/plotly/notebook.ipynb +++ b/examples/plotly/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/plotly/requirements.txt b/examples/plotly/requirements.txt index 84b849067..f26b252b1 100644 --- a/examples/plotly/requirements.txt +++ b/examples/plotly/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton[visualization] plotly -sf-hamilton[visualization] diff --git a/examples/polars/README.md b/examples/polars/README.md index 8ae20344d..7141e6686 100644 --- a/examples/polars/README.md +++ b/examples/polars/README.md @@ -51,7 +51,7 @@ Here is the graph of execution - which should look the same as the pandas exampl There is one major caveat with Polars to be aware of: THERE IS NO INDEX IN POLARS LIKE THERE IS WITH PANDAS. What this means is that when you tell Apache Hamilton to execute and return a polars dataframe if you are using the -[provided results builder](https://github.com/apache/hamilton/blob/sf-hamilton-1.14.1/hamilton/plugins/h_polars.py#L8), i.e. `hamilton.plugins.h_polars.PolarsResultsBuilder`, then you will have to +[provided results builder](https://github.com/apache/hamilton/blob/apache-hamilton-1.14.1/hamilton/plugins/h_polars.py#L8), i.e. `hamilton.plugins.h_polars.PolarsResultsBuilder`, then you will have to ensure the row order matches the order you expect for all the outputs you request. E.g. if you do a filter, or a sort, or a join, or a groupby, you will have to ensure that when you ask Apache Hamilton to materialize an output that it's in the order you expect. diff --git a/examples/polars/lazyframe/README.md b/examples/polars/lazyframe/README.md index 211d6f111..217568189 100644 --- a/examples/polars/lazyframe/README.md +++ b/examples/polars/lazyframe/README.md @@ -45,7 +45,7 @@ Here is the graph of execution - which should look the same as the pandas exampl There is one major caveat with Polars to be aware of: THERE IS NO INDEX IN POLARS LIKE THERE IS WITH PANDAS. What this means is that when you tell Apache Hamilton to execute and return a polars dataframe if you are using the -[provided results builder](https://github.com/apache/hamilton/blob/sf-hamilton-1.14.1/hamilton/plugins/h_polars.py#L8), i.e. `hamilton.plugins.h_polars.PolarsResultsBuilder`, then you will have to +[provided results builder](https://github.com/apache/hamilton/blob/apache-hamilton-1.14.1/hamilton/plugins/h_polars.py#L8), i.e. `hamilton.plugins.h_polars.PolarsResultsBuilder`, then you will have to ensure the row order matches the order you expect for all the outputs you request. E.g. if you do a filter, or a sort, or a join, or a groupby, you will have to ensure that when you ask Apache Hamilton to materialize an output that it's in the order you expect. diff --git a/examples/polars/lazyframe/my_script.py b/examples/polars/lazyframe/my_script.py index 3c51cdbc2..f4fba9e99 100644 --- a/examples/polars/lazyframe/my_script.py +++ b/examples/polars/lazyframe/my_script.py @@ -45,6 +45,6 @@ # And print the table. print(df) -# To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work +# To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './polars', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/polars/lazyframe/requirements.txt b/examples/polars/lazyframe/requirements.txt index 3874d6b99..6791ad22b 100644 --- a/examples/polars/lazyframe/requirements.txt +++ b/examples/polars/lazyframe/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton polars -sf-hamilton diff --git a/examples/polars/materialization/notebook.ipynb b/examples/polars/materialization/notebook.ipynb index 4826d1f26..535b2a318 100644 --- a/examples/polars/materialization/notebook.ipynb +++ b/examples/polars/materialization/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] polars" + "%pip install apache-hamilton[visualization] polars" ] }, { diff --git a/examples/polars/materialization/requirements.txt b/examples/polars/materialization/requirements.txt index f7ae9660e..c0c8d42df 100644 --- a/examples/polars/materialization/requirements.txt +++ b/examples/polars/materialization/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton polars -sf-hamilton xlsx2csv # for excel data loader xlsxwriter # Excel export requires 'xlsxwriter' diff --git a/examples/polars/my_script.py b/examples/polars/my_script.py index aa206b4d9..e86800a7b 100644 --- a/examples/polars/my_script.py +++ b/examples/polars/my_script.py @@ -43,6 +43,6 @@ df = dr.execute(output_columns) print(df) -# To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work +# To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, './polars', {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') diff --git a/examples/polars/notebook.ipynb b/examples/polars/notebook.ipynb index e8a041ca7..67de1d328 100644 --- a/examples/polars/notebook.ipynb +++ b/examples/polars/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization] polars" + "%pip install apache-hamilton[visualization] polars" ] }, { @@ -569,7 +569,7 @@ "df = dr.execute(output_columns, inputs=inputs)\n", "print(df)\n", "\n", - "# To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + "# To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", "dr.visualize_execution(output_columns, inputs=inputs)" ] }, diff --git a/examples/polars/requirements.txt b/examples/polars/requirements.txt index 3874d6b99..6791ad22b 100644 --- a/examples/polars/requirements.txt +++ b/examples/polars/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton polars -sf-hamilton diff --git a/examples/polars/with_columns/notebook.ipynb b/examples/polars/with_columns/notebook.ipynb index 4300694ca..279630913 100644 --- a/examples/polars/with_columns/notebook.ipynb +++ b/examples/polars/with_columns/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/prefect/hamilton_prefect_notebook.ipynb b/examples/prefect/hamilton_prefect_notebook.ipynb index 68da14dc6..14cfe22f6 100644 --- a/examples/prefect/hamilton_prefect_notebook.ipynb +++ b/examples/prefect/hamilton_prefect_notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/prefect/requirements.txt b/examples/prefect/requirements.txt index a9a64fa7b..c943acd79 100644 --- a/examples/prefect/requirements.txt +++ b/examples/prefect/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton[visualization] numpy pandas prefect scikit-learn -sf-hamilton[visualization] diff --git a/examples/ray/hello_world/README.md b/examples/ray/hello_world/README.md index 0539cf496..c4802691b 100644 --- a/examples/ray/hello_world/README.md +++ b/examples/ray/hello_world/README.md @@ -23,7 +23,7 @@ Here we have a hello world example showing how you can take some Apache Hamilton functions and then easily run them in a distributed setting via ray. -`pip install sf-hamilton[ray]` or `pip install sf-hamilton ray` to for the right dependencies to run this example. +`pip install apache-hamilton[ray]` or `pip install apache-hamilton ray` to for the right dependencies to run this example. File organization: diff --git a/examples/ray/hello_world/notebook.ipynb b/examples/ray/hello_world/notebook.ipynb index a2fa436ad..0fa13a5f3 100644 --- a/examples/ray/hello_world/notebook.ipynb +++ b/examples/ray/hello_world/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -395,7 +395,7 @@ } ], "source": [ - "# To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + "# To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", "dr.visualize_execution(output_columns)" ] }, diff --git a/examples/ray/hello_world/run.py b/examples/ray/hello_world/run.py index 17cbd9305..db987a633 100644 --- a/examples/ray/hello_world/run.py +++ b/examples/ray/hello_world/run.py @@ -48,7 +48,7 @@ ] # let's create the dataframe! df = dr.execute(output_columns) - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, "./ray_dag", {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') print(df.to_string()) diff --git a/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb b/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb index fc996577c..9e4da2eb5 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb +++ b/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/ray/ray_Hamilton_UI_tracking/requirements.txt b/examples/ray/ray_Hamilton_UI_tracking/requirements.txt index e6916bb92..1cbb533f8 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/requirements.txt +++ b/examples/ray/ray_Hamilton_UI_tracking/requirements.txt @@ -1 +1 @@ -sf-hamilton[ray,sdk,ui] +apache-hamilton[ray,sdk,ui] diff --git a/examples/ray/requirements.txt b/examples/ray/requirements.txt index 7d0448121..8490c36ab 100644 --- a/examples/ray/requirements.txt +++ b/examples/ray/requirements.txt @@ -1 +1 @@ -sf-hamilton[ray] +apache-hamilton[ray] diff --git a/examples/reusing_functions/reusing_functions.ipynb b/examples/reusing_functions/reusing_functions.ipynb index 4047eb347..1cbcf3c8d 100644 --- a/examples/reusing_functions/reusing_functions.ipynb +++ b/examples/reusing_functions/reusing_functions.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -57,7 +57,7 @@ "metadata": {}, "outputs": [], "source": [ - "# %pip install sf-hamilton pandas" + "# %pip install apache-hamilton pandas" ] }, { diff --git a/examples/reverse_etl/notebook.ipynb b/examples/reverse_etl/notebook.ipynb index 769c2bb9d..aa8c7fa95 100644 --- a/examples/reverse_etl/notebook.ipynb +++ b/examples/reverse_etl/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/reverse_etl/requirements.txt b/examples/reverse_etl/requirements.txt index 5f13366a9..47cc1c79f 100644 --- a/examples/reverse_etl/requirements.txt +++ b/examples/reverse_etl/requirements.txt @@ -1,5 +1,5 @@ +apache-hamilton[visualization] google-api-python-client google-auth pandas -sf-hamilton[visualization] timew-report diff --git a/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb b/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb index 3f71e45c1..bca8c1564 100644 --- a/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb +++ b/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/scikit-learn/transformer/hamilton_notebook.ipynb b/examples/scikit-learn/transformer/hamilton_notebook.ipynb index ec057a9ce..4cbe4cc73 100644 --- a/examples/scikit-learn/transformer/hamilton_notebook.ipynb +++ b/examples/scikit-learn/transformer/hamilton_notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -446,7 +446,7 @@ ], "source": [ "# Visualize execution\n", - "# To visualize do `pip install \"sf-hamilton[visualization]\"` if you want these to work\n", + "# To visualize do `pip install \"apache-hamilton[visualization]\"` if you want these to work\n", "\n", "# visualize all possible functions\n", "dr.display_all_functions(output_file_path=None)" diff --git a/examples/scikit-learn/transformer/requirements.txt b/examples/scikit-learn/transformer/requirements.txt index 3f69ad5c2..559a0b7c0 100644 --- a/examples/scikit-learn/transformer/requirements.txt +++ b/examples/scikit-learn/transformer/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton scikit-learn -sf-hamilton diff --git a/examples/slack/notebook.ipynb b/examples/slack/notebook.ipynb index e6f075f3a..e348dea58 100644 --- a/examples/slack/notebook.ipynb +++ b/examples/slack/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization,slack]" + "%pip install apache-hamilton[visualization,slack]" ] }, { diff --git a/examples/slack/requirements.txt b/examples/slack/requirements.txt index bbcc25185..87f593059 100644 --- a/examples/slack/requirements.txt +++ b/examples/slack/requirements.txt @@ -1 +1 @@ -sf-hamilton[slack] +apache-hamilton[slack] diff --git a/examples/snowflake/hamilton_ui/requirements.txt b/examples/snowflake/hamilton_ui/requirements.txt index 7e573a31a..cc7f94e2f 100644 --- a/examples/snowflake/hamilton_ui/requirements.txt +++ b/examples/snowflake/hamilton_ui/requirements.txt @@ -1,2 +1,2 @@ +apache-hamilton[ui,sdk] flask -sf-hamilton[ui,sdk] diff --git a/examples/spark/pandas_on_spark/README.md b/examples/spark/pandas_on_spark/README.md index dd11e85f9..dd48e2fbd 100644 --- a/examples/spark/pandas_on_spark/README.md +++ b/examples/spark/pandas_on_spark/README.md @@ -23,7 +23,7 @@ Here we have a hello world example showing how you can take some Apache Hamilton functions and then easily run them in a distributed setting via Spark 3.2+ using Koalas. -`pip install sf-hamilton[pyspark]` or `pip install sf-hamilton pyspark[pandas_on_spark]` to for the right dependencies to run this example. +`pip install apache-hamilton[pyspark]` or `pip install apache-hamilton pyspark[pandas_on_spark]` to for the right dependencies to run this example. File organization: diff --git a/examples/spark/pandas_on_spark/run.py b/examples/spark/pandas_on_spark/run.py index 429685711..33df72b5d 100644 --- a/examples/spark/pandas_on_spark/run.py +++ b/examples/spark/pandas_on_spark/run.py @@ -83,7 +83,7 @@ ] # let's create the dataframe! df = dr.execute(output_columns) - # To visualize do `pip install "sf-hamilton[visualization]"` if you want these to work + # To visualize do `pip install "apache-hamilton[visualization]"` if you want these to work # dr.visualize_execution(output_columns, "./pandas_on_spark", {"format": "png"}) # dr.display_all_functions('./my_full_dag.dot') print(type(df)) diff --git a/examples/spark/pyspark/notebook.ipynb b/examples/spark/pyspark/notebook.ipynb index 91f45d6ba..bf5c81872 100644 --- a/examples/spark/pyspark/notebook.ipynb +++ b/examples/spark/pyspark/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/spark/pyspark_feature_catalog/example_usage.ipynb b/examples/spark/pyspark_feature_catalog/example_usage.ipynb index 8328ec6ff..ad36b7784 100644 --- a/examples/spark/pyspark_feature_catalog/example_usage.ipynb +++ b/examples/spark/pyspark_feature_catalog/example_usage.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/spark/pyspark_udfs/README.md b/examples/spark/pyspark_udfs/README.md index a8d52b7ed..1c693a4c0 100644 --- a/examples/spark/pyspark_udfs/README.md +++ b/examples/spark/pyspark_udfs/README.md @@ -24,7 +24,7 @@ easily run them as pyspark UDFs. ## Dependencies -`pip install "sf-hamilton[pyspark, visualization]"` for the right dependencies to run this example. +`pip install "apache-hamilton[pyspark, visualization]"` for the right dependencies to run this example. ## File organization: diff --git a/examples/spark/requirements.txt b/examples/spark/requirements.txt index 81c5490ce..cd6496d73 100644 --- a/examples/spark/requirements.txt +++ b/examples/spark/requirements.txt @@ -1 +1 @@ -sf-hamilton[pyspark, visualization] +apache-hamilton[pyspark, visualization] diff --git a/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb b/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb index 45d6166fa..a169d14c5 100644 --- a/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb +++ b/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { @@ -320,12 +320,12 @@ "]\n", "\n", "# let's create the dataframe!\n", - "# if you only did `pip install sf-hamilton` earlier:\n", + "# if you only did `pip install apache-hamilton` earlier:\n", "df = dr.execute(\n", " output_columns,\n", " inputs={**input_df.to_dict(orient=\"series\"), \"aggregation_level\": aggregation_level},\n", ")\n", - "# else if you did `pip install \"sf-hamilton[visualization]\"` earlier:\n", + "# else if you did `pip install \"apache-hamilton[visualization]\"` earlier:\n", "# dr.visualize_execution(output_columns, './my-dag.dot', {})\n", "print(df)" ] diff --git a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb index bd63ba02c..d1b901c69 100644 --- a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb +++ b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb index 7d958481e..e0485941c 100644 --- a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb +++ b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/streamlit/requirements.txt b/examples/streamlit/requirements.txt index 037b75b46..c119800d6 100644 --- a/examples/streamlit/requirements.txt +++ b/examples/streamlit/requirements.txt @@ -1,4 +1,4 @@ +apache-hamilton pandas plotly -sf-hamilton streamlit diff --git a/examples/vaex/notebook.ipynb b/examples/vaex/notebook.ipynb index ba4d1b220..a5d5fa051 100644 --- a/examples/vaex/notebook.ipynb +++ b/examples/vaex/notebook.ipynb @@ -14,7 +14,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/examples/vaex/requirements.txt b/examples/vaex/requirements.txt index 974815425..22cc3938d 100644 --- a/examples/vaex/requirements.txt +++ b/examples/vaex/requirements.txt @@ -1 +1 @@ -sf-hamilton[vaex] +apache-hamilton[vaex] diff --git a/examples/validate_examples.py b/examples/validate_examples.py index 16f531c77..cc7719b0c 100644 --- a/examples/validate_examples.py +++ b/examples/validate_examples.py @@ -69,7 +69,7 @@ def validate_notebook(notebook_path: pathlib.Path) -> int: #%% CELL_2 (or CELL_1 if no license) # Execute this cell to install dependencies - %pip install sf-hamilton[visualization] matplotlib + %pip install apache-hamilton[visualization] matplotlib #%% CELL_3 (or CELL_2 if no license) # Title of the notebook ![Colab badge](colab_url) ![GitHub badge](github_url) @@ -161,7 +161,7 @@ def insert_setup_cell(path: pathlib.Path): """ notebook = nbformat.read(path, as_version=4) setup_cell = nbformat.v4.new_code_cell( - "# Execute this cell to install dependencies\n%pip install sf-hamilton[visualization]" + "# Execute this cell to install dependencies\n%pip install apache-hamilton[visualization]" ) # Check if first cell is a license header diff --git a/examples/validation/static_validator/notebook.ipynb b/examples/validation/static_validator/notebook.ipynb index ecf0db6ba..bf2648cbb 100644 --- a/examples/validation/static_validator/notebook.ipynb +++ b/examples/validation/static_validator/notebook.ipynb @@ -16,7 +16,7 @@ "outputs": [], "source": [ "# Execute this cell to install dependencies\n", - "%pip install sf-hamilton[visualization]" + "%pip install apache-hamilton[visualization]" ] }, { diff --git a/hamilton/cli/__main__.py b/hamilton/cli/__main__.py index 891cce442..58d1174fd 100644 --- a/hamilton/cli/__main__.py +++ b/hamilton/cli/__main__.py @@ -309,7 +309,7 @@ def ui( except ImportError as e: logger.error( "hamilton[ui] not installed -- you have to install this to run the UI. " - 'Run `pip install "sf-hamilton[ui]"` to install and get started with the UI!' + 'Run `pip install "apache-hamilton[ui]"` to install and get started with the UI!' ) raise typer.Exit(code=1) from e diff --git a/hamilton/contrib/__init__.py b/hamilton/contrib/__init__.py index f6bc625f3..582497cac 100644 --- a/hamilton/contrib/__init__.py +++ b/hamilton/contrib/__init__.py @@ -15,15 +15,15 @@ # specific language governing permissions and limitations # under the License. -"""This module exists so that people can download dataflows without the sf-hamilton-contrib package. +"""This module exists so that people can download dataflows without the apache-hamilton-contrib package. -It will get clobbered when sf-hamilton-contrib is installed, which is good. +It will get clobbered when apache-hamilton-contrib is installed, which is good. """ import logging from contextlib import contextmanager -__version__ = "__unknown__" # this will be overwritten once sf-hamilton-contrib is installed. +__version__ = "__unknown__" # this will be overwritten once apache-hamilton-contrib is installed. @contextmanager diff --git a/hamilton/data_quality/default_validators.py b/hamilton/data_quality/default_validators.py index a01fcab02..b0ea1e95f 100644 --- a/hamilton/data_quality/default_validators.py +++ b/hamilton/data_quality/default_validators.py @@ -515,7 +515,7 @@ def _append_pandera_to_default_validators(): import pandera # noqa: F401 except ModuleNotFoundError: logger.info( - "Cannot import pandera from pandera_validators. Run pip install sf-hamilton[pandera] if needed." + "Cannot import pandera from pandera_validators. Run pip install apache-hamilton[pandera] if needed." ) return from hamilton.data_quality import pandera_validators diff --git a/hamilton/graph.py b/hamilton/graph.py index f3fac7558..4a9ba471a 100644 --- a/hamilton/graph.py +++ b/hamilton/graph.py @@ -885,7 +885,7 @@ def get_cycles(self, nodes: set[node.Node], user_nodes: set[node.Node]) -> list[ except ModuleNotFoundError: logger.exception( " networkx is required for detecting cycles in the function graph. Install it with:" - '\n\n pip install "sf-hamilton[visualization]" or pip install networkx \n\n' + '\n\n pip install "apache-hamilton[visualization]" or pip install networkx \n\n' ) return [] digraph = create_networkx_graph(nodes, user_nodes, "Dependency Graph") @@ -946,7 +946,7 @@ def display( except ModuleNotFoundError: logger.exception( " graphviz is required for visualizing the function graph. Install it with:" - '\n\n pip install "sf-hamilton[visualization]" or pip install graphviz \n\n' + '\n\n pip install "apache-hamilton[visualization]" or pip install graphviz \n\n' ) return if graphviz_kwargs is None: diff --git a/hamilton/plugins/h_dask.py b/hamilton/plugins/h_dask.py index 2e2c2da96..157aca63a 100644 --- a/hamilton/plugins/h_dask.py +++ b/hamilton/plugins/h_dask.py @@ -44,7 +44,7 @@ class DaskGraphAdapter(base.HamiltonGraphAdapter): This walks the graph and translates it to run onto `Dask `__. - Use `pip install sf-hamilton[dask]` to get the dependencies required to run this. + Use `pip install apache-hamilton[dask]` to get the dependencies required to run this. Try this adapter when: diff --git a/hamilton/plugins/h_experiments/__main__.py b/hamilton/plugins/h_experiments/__main__.py index bb54be4d6..19f233ad1 100644 --- a/hamilton/plugins/h_experiments/__main__.py +++ b/hamilton/plugins/h_experiments/__main__.py @@ -29,7 +29,7 @@ def main(): import uvicorn except ModuleNotFoundError as e: raise ModuleNotFoundError( - "Some dependencies are missing. Make sure to `pip install sf-hamilton[experiments]`" + "Some dependencies are missing. Make sure to `pip install apache-hamilton[experiments]`" ) from e parser = argparse.ArgumentParser(prog="hamilton-experiments") parser.description = "Hamilton Experiment Server launcher" diff --git a/hamilton/plugins/h_opentelemetry.py b/hamilton/plugins/h_opentelemetry.py index fc73892a7..1d6875619 100644 --- a/hamilton/plugins/h_opentelemetry.py +++ b/hamilton/plugins/h_opentelemetry.py @@ -30,7 +30,7 @@ except ImportError as e: raise ImportError( "Failed to import `opentelemetry` " - "Use `pip install sf-hamilton[opentelemetry]` to install " + "Use `pip install apache-hamilton[opentelemetry]` to install " "dependencies for the `h_opentelemetry` plugin." ) from e diff --git a/hamilton/plugins/h_ray.py b/hamilton/plugins/h_ray.py index 05bfcf402..cd8b2d85c 100644 --- a/hamilton/plugins/h_ray.py +++ b/hamilton/plugins/h_ray.py @@ -80,7 +80,7 @@ class RayGraphAdapter( This walks the graph and translates it to run onto `Ray `__. - Use `pip install sf-hamilton[ray]` to get the dependencies required to run this. + Use `pip install apache-hamilton[ray]` to get the dependencies required to run this. Use this if: diff --git a/hamilton/plugins/h_rich.py b/hamilton/plugins/h_rich.py index 3eb777aaf..d9729aab6 100644 --- a/hamilton/plugins/h_rich.py +++ b/hamilton/plugins/h_rich.py @@ -42,7 +42,7 @@ class RichProgressBar(TaskExecutionHook, TaskGroupingHook, GraphExecutionHook, N """An adapter that uses rich to show simple progress bars for the graph execution. Note: you need to have rich installed for this to work. If you don't have it installed, you can - install it with `pip install rich` (or `pip install sf-hamilton[rich]` -- use quotes if you're + install it with `pip install rich` (or `pip install apache-hamilton[rich]` -- use quotes if you're using zsh). .. code-block:: python diff --git a/hamilton/plugins/h_slack.py b/hamilton/plugins/h_slack.py index 28a65e663..1114119f3 100644 --- a/hamilton/plugins/h_slack.py +++ b/hamilton/plugins/h_slack.py @@ -29,7 +29,7 @@ class SlackNotifier(NodeExecutionHook): Note: you need to have slack_sdk installed for this to work. If you don't have it installed, you can install it with `pip install slack_sdk` - (or `pip install sf-hamilton[slack]` -- use quotes if you're using zsh). + (or `pip install apache-hamilton[slack]` -- use quotes if you're using zsh). .. code-block:: python diff --git a/hamilton/plugins/h_spark.py b/hamilton/plugins/h_spark.py index 1033f1371..84474ecc1 100644 --- a/hamilton/plugins/h_spark.py +++ b/hamilton/plugins/h_spark.py @@ -61,7 +61,7 @@ class SparkKoalasGraphAdapter(base.HamiltonGraphAdapter, base.ResultMixin): using the \ `Pandas API on Spark `__ - Use `pip install sf-hamilton[spark]` to get the dependencies required to run this. + Use `pip install apache-hamilton[spark]` to get the dependencies required to run this. Currently, this class assumes you're running SPARK 3.2+. You'd generally use this if you have an existing spark \ cluster running in your workplace, and you want to scale to very large data set sizes. diff --git a/hamilton/plugins/h_tqdm.py b/hamilton/plugins/h_tqdm.py index 016b1ec29..0c734831d 100644 --- a/hamilton/plugins/h_tqdm.py +++ b/hamilton/plugins/h_tqdm.py @@ -32,7 +32,7 @@ class ProgressBar( Note: you need to have tqdm installed for this to work. If you don't have it installed, you can install it with `pip install tqdm` - (or `pip install sf-hamilton[tqdm]` -- use quotes if you're using zsh). + (or `pip install apache-hamilton[tqdm]` -- use quotes if you're using zsh). .. code-block:: python diff --git a/hamilton/plugins/pydantic_extensions.py b/hamilton/plugins/pydantic_extensions.py index e0029e28e..cef920a9b 100644 --- a/hamilton/plugins/pydantic_extensions.py +++ b/hamilton/plugins/pydantic_extensions.py @@ -24,14 +24,14 @@ import pydantic # noqa: F401 except ModuleNotFoundError as e: raise NotImplementedError( - "Cannot import `pydantic` from `pydantic_validators`. Run pip install 'sf-hamilton[pydantic]' if needed." + "Cannot import `pydantic` from `pydantic_validators`. Run pip install 'apache-hamilton[pydantic]' if needed." ) from e try: from pydantic import BaseModel, TypeAdapter, ValidationError except ImportError as e: raise NotImplementedError( - "`pydantic>=2.0` required to use `pydantic_validators`. Run pip install 'sf-hamilton[pydantic]' if needed." + "`pydantic>=2.0` required to use `pydantic_validators`. Run pip install 'apache-hamilton[pydantic]' if needed." ) from e diff --git a/scripts/find_hanging_tests.sh b/scripts/find_hanging_tests.sh new file mode 100755 index 000000000..8f9dbedb3 --- /dev/null +++ b/scripts/find_hanging_tests.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Finds tests that prevent pytest from shutting down cleanly. +# Runs each test file individually with a timeout and reports any that hang. +# +# Usage: +# ./find_hanging_tests.sh [timeout_seconds] [test_directory] +# ./find_hanging_tests.sh # defaults: 30s, tests/ +# ./find_hanging_tests.sh 60 tests/plugins + +set -uo pipefail + +TIMEOUT="${1:-30}" +TEST_DIR="${2:-tests/}" +HANGING=() +FAILED=() +PASSED=() + +echo "Finding test files in ${TEST_DIR}..." +TEST_FILES=$(find "$TEST_DIR" -name "test_*.py" -type f | sort) +TOTAL=$(echo "$TEST_FILES" | wc -l | tr -d ' ') + +echo "Found ${TOTAL} test files. Timeout per file: ${TIMEOUT}s" +echo "" + +i=0 +for test_file in $TEST_FILES; do + i=$((i + 1)) + printf "[%3d/%d] %-70s " "$i" "$TOTAL" "$test_file" + + timeout "$TIMEOUT" uv run pytest "$test_file" -x -q 2>&1 > /tmp/hanging_test_output.txt + exit_code=$? + + if [ $exit_code -eq 124 ]; then + echo "HANG (killed after ${TIMEOUT}s)" + HANGING+=("$test_file") + elif [ $exit_code -ne 0 ]; then + echo "FAIL (exit $exit_code)" + FAILED+=("$test_file") + else + echo "OK" + PASSED+=("$test_file") + fi +done + +echo "" +echo "================================================================" +echo " Results" +echo "================================================================" +echo "" +echo "Passed: ${#PASSED[@]}" +echo "Failed: ${#FAILED[@]}" +echo "Hanging: ${#HANGING[@]}" + +if [ ${#FAILED[@]} -gt 0 ]; then + echo "" + echo "--- Failed tests ---" + for f in "${FAILED[@]}"; do + echo " $f" + done +fi + +if [ ${#HANGING[@]} -gt 0 ]; then + echo "" + echo "--- Hanging tests (did not exit within ${TIMEOUT}s) ---" + for f in "${HANGING[@]}"; do + echo " $f" + done +fi diff --git a/ui/README.md b/ui/README.md index 29dd10f4e..4d54b76ca 100644 --- a/ui/README.md +++ b/ui/README.md @@ -122,10 +122,10 @@ This: 1. Builds the frontend 2. Copies it into the build/ directory -3. Publishes to the [sf-hamilton-ui](https://pypi.org/project/sf-hamilton-ui/) package on pypi +3. Publishes to the [apache-hamilton-ui](https://pypi.org/project/apache-hamilton-ui/) package on pypi -Then you'll run it with `hamilton ui` after installing `sf-hamilton[ui]`. Note to -talk to it you'll need the hamilton_sdk pacakge which can be installed with `pip install sf-hamilton[sdk]`. +Then you'll run it with `hamilton ui` after installing `apache-hamilton[ui]`. Note to +talk to it you'll need the hamilton_sdk pacakge which can be installed with `pip install apache-hamilton[sdk]`. ## Building docker diff --git a/ui/backend/pyproject.toml b/ui/backend/pyproject.toml index 0e20fff78..1a8259fbd 100644 --- a/ui/backend/pyproject.toml +++ b/ui/backend/pyproject.toml @@ -56,8 +56,7 @@ dependencies = [ "psycopg2-binary", "pydantic>=2.0.0", "requests>=2.32.0", - # "apache-hamilton>=1.58.0", # TODO once we publish properly fix this. - "sf-hamilton>=1.58.0", + "apache-hamilton>=1.58.0", "typer", ] diff --git a/ui/backend/server/requirements-base.txt b/ui/backend/server/requirements-base.txt index 51053cd74..675e1ba32 100644 --- a/ui/backend/server/requirements-base.txt +++ b/ui/backend/server/requirements-base.txt @@ -2,6 +2,7 @@ aioboto3 aiobotocore aiofiles aiohttp +apache-hamilton colorlog ddtrace django @@ -11,4 +12,3 @@ passlib propelauth_py psycopg2-binary pydantic -sf-hamilton diff --git a/ui/frontend/src/components/dashboard/Project/ProjectLogInstructions.tsx b/ui/frontend/src/components/dashboard/Project/ProjectLogInstructions.tsx index 180402c71..af44e5481 100644 --- a/ui/frontend/src/components/dashboard/Project/ProjectLogInstructions.tsx +++ b/ui/frontend/src/components/dashboard/Project/ProjectLogInstructions.tsx @@ -26,7 +26,7 @@ import { classNames } from "../../../utils"; import { CheckBox } from "../../common/Checkbox"; const INSTALL_CODE = `# be sure to initialize any python environment you want to use first -pip install "sf-hamilton[sdk]"`; +pip install "apache-hamilton[sdk]"`; const INSTALL_CODE_DW = `# be sure to initialize any python environment you want to use first pip install dagworks-sdk`; diff --git a/ui/sdk/README.md b/ui/sdk/README.md index af067b3bf..6cc9591ea 100644 --- a/ui/sdk/README.md +++ b/ui/sdk/README.md @@ -46,7 +46,7 @@ can do this with: ```bash # install the package & cli into your favorite python environment. -pip install "sf-hamilton[sdk]" +pip install "apache-hamilton[sdk]" # And validate -- this should not error. python -c "from hamilton_sdk import adapters" diff --git a/ui/sdk/src/hamilton_sdk/cli/templates/data_processing/template_data.json b/ui/sdk/src/hamilton_sdk/cli/templates/data_processing/template_data.json index 8e1adb420..0f6dffc70 100644 --- a/ui/sdk/src/hamilton_sdk/cli/templates/data_processing/template_data.json +++ b/ui/sdk/src/hamilton_sdk/cli/templates/data_processing/template_data.json @@ -4,7 +4,7 @@ "return_df": true, "inputs" : {"orders_path": "data/orders_old.csv", "order_details_path": "data/order_details.csv"}, - "requirements" : ["pandas", "sf-hamilton", "pandera"], + "requirements" : ["pandas", "apache-hamilton", "pandera"], "use_tags_for_output" : null, "final_vars" : [ "orders_by_order_aggregates" diff --git a/ui/sdk/src/hamilton_sdk/cli/templates/hello_world/template_data.json b/ui/sdk/src/hamilton_sdk/cli/templates/hello_world/template_data.json index 04b457c0e..75f72424d 100644 --- a/ui/sdk/src/hamilton_sdk/cli/templates/hello_world/template_data.json +++ b/ui/sdk/src/hamilton_sdk/cli/templates/hello_world/template_data.json @@ -6,7 +6,7 @@ "signups_path" : "data/signups.csv", "spend_path" : "data/spend.csv" }, - "requirements" : ["pandas", "sf-hamilton"], + "requirements" : ["pandas", "apache-hamilton"], "use_tags_for_output" : null, "final_vars" : [ "spend", diff --git a/ui/sdk/src/hamilton_sdk/cli/templates/machine_learning/template_data.json b/ui/sdk/src/hamilton_sdk/cli/templates/machine_learning/template_data.json index 5ef1531d8..782aa9ebb 100644 --- a/ui/sdk/src/hamilton_sdk/cli/templates/machine_learning/template_data.json +++ b/ui/sdk/src/hamilton_sdk/cli/templates/machine_learning/template_data.json @@ -3,7 +3,7 @@ "configs" : [], "return_df": false, "inputs" : {}, - "requirements" : ["pandas", "sf-hamilton", "scikit-learn"], + "requirements" : ["pandas", "apache-hamilton", "scikit-learn"], "use_tags_for_output" : null, "final_vars" : [ "best_model" diff --git a/ui/sdk/src/hamilton_sdk/cli/templates/time_series_feature_engineering/template_data.json b/ui/sdk/src/hamilton_sdk/cli/templates/time_series_feature_engineering/template_data.json index 32a178ecf..99308d0c1 100644 --- a/ui/sdk/src/hamilton_sdk/cli/templates/time_series_feature_engineering/template_data.json +++ b/ui/sdk/src/hamilton_sdk/cli/templates/time_series_feature_engineering/template_data.json @@ -5,7 +5,7 @@ "data_path": "data/train_sample.csv" }, "return_df": true, - "requirements" : ["pandas", "sf-hamilton"], + "requirements" : ["pandas", "apache-hamilton"], "use_tags_for_output" : ["stage", "production"], "final_vars" : null, "template_description": "This is an example showing how one might start approaching time-series feature engineering. It is based on code from https://github.com/F33AI/waffleml-examples/tree/master/kaggle_store_item_demand_forecasting. It uses a sample of data from this kaggle competition - https://www.kaggle.com/competitions/demand-forecasting-kernels-only." diff --git a/writeups/data_quality.md b/writeups/data_quality.md index 9cb857906..d7fabc5f3 100644 --- a/writeups/data_quality.md +++ b/writeups/data_quality.md @@ -63,7 +63,7 @@ We've fully integrated data quality with [pandera](https://pandera.readthedocs.i Note that you have to have hamilton installed with the `pandera` extension. E.G. ```bash -pip install sf-hamilton[pandera] +pip install apache-hamilton[pandera] ``` The integration point is simple. All you have to do is provide a pandera schema diff --git a/writeups/developer_setup.md b/writeups/developer_setup.md index 2f82b2281..df0079154 100644 --- a/writeups/developer_setup.md +++ b/writeups/developer_setup.md @@ -155,7 +155,7 @@ that are setup to run each time you push a commit to a branch in github. # Pushing to pypi These are the steps to push to pypi. This is taken from the [python packaging tutorial](https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives). -1. Have an account & be granted the ability to push to sf-hamilton on testpypi & real pypi. +1. Have an account & be granted the ability to push to apache-hamilton on testpypi & real pypi. 2. Setup API tokens and add them to your ~/.pypirc. 3. Run `python3 -m pip install --upgrade build`. 4. Run `python3 -m pip install --upgrade twine` @@ -173,8 +173,8 @@ These are the steps to push to pypi. This is taken from the [python packaging tu Now that you've pushed to pypi, announce your release! We plan to automate this, but for now... -1. Create a github release (select auto-generate release for painless text generation). Create a tag that's called `sf-hamilton-{version_string}`. -See [1.2.0](https://github.com/apache/hamilton/releases/tag/sf-hamilton-1.2.0) for an example. +1. Create a github release (select auto-generate release for painless text generation). Create a tag that's called `apache-hamilton-{version_string}`. +See [1.2.0](https://github.com/apache/hamilton/releases/tag/apache-hamilton-1.2.0) for an example. 2. Announce on the #announcements channel in slack, following the format presented there. 3. Thanks for contributing! From 3bb18b7bac4237879b2b91c2e04dbfcc328dad14 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sat, 25 Apr 2026 21:01:16 -0700 Subject: [PATCH 2/4] Update pyproject.toml extras and disable ddtrace pytest plugin - Point lsp/sdk/ui extras to sf-hamilton-* with TODO to switch once stable apache-hamilton-* versions are published to PyPI - Update ui/backend dependency from sf-hamilton to apache-hamilton - Disable ddtrace pytest plugin which spawns background threads that prevent clean process exit after the test suite completes --- pyproject.toml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f6d059bd1..1dac0296a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ experiments = [ "fastui", "uvicorn", ] -lsp = ["sf-hamilton-lsp"] +lsp = ["sf-hamilton-lsp"] # TODO: switch to apache-hamilton-lsp once stable version is published openlineage = ["openlineage-python"] pandera = ["pandera"] pydantic = ["pydantic>=2.0"] @@ -76,12 +76,12 @@ pyspark = [ ] ray = ["ray>=2.0.0; python_version < '3.14'", "pyarrow"] rich = ["rich"] -sdk = ["sf-hamilton-sdk"] +sdk = ["sf-hamilton-sdk"] # TODO: switch to apache-hamilton-sdk once stable version is published slack = ["slack-sdk"] mcp = ["fastmcp>=3.0.0,<4"] tqdm = ["tqdm"] -ui = ["sf-hamilton-ui"] +ui = ["sf-hamilton-ui"] # TODO: switch to apache-hamilton-ui once stable version is published # vaex -- on >=py3.11 only core part available https://github.com/vaexio/vaex/pull/2331#issuecomment-2437198176 vaex = [ @@ -200,9 +200,8 @@ slack = "https://join.slack.com/t/hamilton-opensource/shared_invite/zt-2niepkra8 #[tool.mypy] #exclude = [] -#[tool.pytest.ini_options] -#pythonpath = [] -#testpaths = [] +[tool.pytest.ini_options] +addopts = "-p no:ddtrace" [tool.ruff] line-length = 100 From 585ccaa3ac22a237689fe493e2679fcc397abf9d Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sat, 25 Apr 2026 21:32:54 -0700 Subject: [PATCH 3/4] Fix docusaurus CI build by switching from yarn to npm The contrib/docs project uses npm (has package-lock.json, no yarn.lock) but the workflow was using yarn with cache: yarn. This caused webpack Progress Plugin validation errors. Switch to npm ci for deterministic installs matching the lockfile. --- .github/workflows/docusaurus-gh-pages.yml | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docusaurus-gh-pages.yml b/.github/workflows/docusaurus-gh-pages.yml index 55f1af159..a3e863a03 100644 --- a/.github/workflows/docusaurus-gh-pages.yml +++ b/.github/workflows/docusaurus-gh-pages.yml @@ -34,7 +34,6 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 1000 - # 👇 Build steps - name: Set up Python 3.10 uses: actions/setup-python@v4 with: @@ -51,19 +50,14 @@ jobs: uses: actions/setup-node@v3 with: node-version: 20.x - cache: yarn - # The action defaults to search for the dependency file - # (package-lock.json or yarn.lock) in the repository root, and uses - # its hash as a part of the cache key. - # https://github.com/actions/setup-node#caching-packages-dependencies + cache: npm cache-dependency-path: "./contrib/docs/package-lock.json" - name: Install dependencies working-directory: contrib/docs - run: yarn install --frozen-lockfile --non-interactive + run: npm ci - name: Build working-directory: contrib/docs - run: yarn build - # 👆 Build steps + run: npm run build deploy: if: ${{ github.ref == 'refs/heads/main' }} environment: @@ -92,19 +86,14 @@ jobs: uses: actions/setup-node@v3 with: node-version: 20.x - cache: yarn - # The action defaults to search for the dependency file - # (package-lock.json or yarn.lock) in the repository root, and uses - # its hash as a part of the cache key. - # https://github.com/actions/setup-node#caching-packages-dependencies + cache: npm cache-dependency-path: "./contrib/docs/package-lock.json" - name: Install dependencies working-directory: contrib/docs - run: yarn install --frozen-lockfile --non-interactive + run: npm ci - name: Build working-directory: contrib/docs - run: yarn build - # 👆 Build steps + run: npm run build - name: Setup Pages uses: actions/configure-pages@v3 - name: Upload artifact From 52227c8e437de48b1facad3077185d42a28f3170 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sat, 16 May 2026 11:27:55 -0700 Subject: [PATCH 4/4] Address review: rename remaining sf-hamilton references in Dockerfile, conda script, and jinja2 template --- .../docs/templates/driver_builder.py.jinja2 | 6 +-- examples/snowflake/hamilton_ui/Dockerfile | 2 +- scripts/build_conda.sh | 4 +- ui/backend/pyproject.toml | 4 +- ui/backend/uv.lock | 39 +++++++++---------- 5 files changed, 27 insertions(+), 28 deletions(-) diff --git a/contrib/docs/templates/driver_builder.py.jinja2 b/contrib/docs/templates/driver_builder.py.jinja2 index 89f98e7d1..a7923edaf 100644 --- a/contrib/docs/templates/driver_builder.py.jinja2 +++ b/contrib/docs/templates/driver_builder.py.jinja2 @@ -11,10 +11,10 @@ from hamilton.execution import executors {%- endif %} {%- else %} {%- if is_user %} -# Make sure you've done - `pip install sf-hamilton-contrib --upgrade` +# Make sure you've done - `pip install apache-hamilton-contrib --upgrade` from hamilton.contrib.user.{{USER}} import {{MODULE_NAME}} {%- else %} -# Make sure you've done - `pip install sf-hamilton-contrib --upgrade` +# Make sure you've done - `pip install apache-hamilton-contrib --upgrade` from hamilton.contrib.dagworks import {{MODULE_NAME}} {%- endif %} {%- endif %} @@ -32,7 +32,7 @@ dr = ( .with_modules({{MODULE_NAME}}) .build() ) -# If you have sf-hamilton[visualization] installed, you can see the dataflow graph +# If you have apache-hamilton[visualization] installed, you can see the dataflow graph # In a notebook this will show an image, else pass in arguments to save to a file # dr.display_all_functions() # Execute the dataflow, specifying what you want back. Will return a dictionary. diff --git a/examples/snowflake/hamilton_ui/Dockerfile b/examples/snowflake/hamilton_ui/Dockerfile index a71115660..126b3c19f 100644 --- a/examples/snowflake/hamilton_ui/Dockerfile +++ b/examples/snowflake/hamilton_ui/Dockerfile @@ -19,7 +19,7 @@ FROM python:3.13.1-slim RUN apt update RUN apt upgrade sqlite3 -y -RUN pip install "sf-hamilton[ui,sdk]" +RUN pip install "apache-hamilton[ui,sdk]" RUN pip install flask ENV HAMILTON_ALLOWED_HOSTS=".snowflakecomputing.app" diff --git a/scripts/build_conda.sh b/scripts/build_conda.sh index d56af6992..f76af47ee 100644 --- a/scripts/build_conda.sh +++ b/scripts/build_conda.sh @@ -20,12 +20,12 @@ CONDA_HOME=$HOME/anaconda3 # This is a script to create and build hamilton for conda. # Be sure you have conda activated and have logged into anaconda # conda activate && anaconda login -pkg='sf-hamilton' +pkg='apache-hamilton' # adjust the Python versions you would like to build array=(3.10 3.11 3.12 3.13) echo "Building conda package ..." cd ~ -# this will create a ~/sf-hamilton directory with metadata to build the package. +# this will create a ~/apache-hamilton directory with metadata to build the package. # will error if it already exists. conda skeleton pypi $pkg diff --git a/ui/backend/pyproject.toml b/ui/backend/pyproject.toml index 1a8259fbd..b52e62cf1 100644 --- a/ui/backend/pyproject.toml +++ b/ui/backend/pyproject.toml @@ -23,7 +23,7 @@ build-backend = "flit_core.buildapi" name = "apache-hamilton-ui" version = "0.0.17" description = "Apache Hamilton UI tracking server for dataflow visualization and monitoring." -requires-python = ">=3.10, <4" +requires-python = ">=3.10.1, <4" license = "Apache-2.0" license-files = ["LICENSE", "NOTICE", "DISCLAIMER"] keywords = ["hamilton", "ui", "tracking", "visualization"] @@ -56,7 +56,7 @@ dependencies = [ "psycopg2-binary", "pydantic>=2.0.0", "requests>=2.32.0", - "apache-hamilton>=1.58.0", + "apache-hamilton>=1.90.0", "typer", ] diff --git a/ui/backend/uv.lock b/ui/backend/uv.lock index 7f543556b..f3d5330f9 100644 --- a/ui/backend/uv.lock +++ b/ui/backend/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.10, <4" +requires-python = ">=3.10.1, <4" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", @@ -186,6 +186,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "apache-hamilton" +version = "1.90.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions" }, + { name = "typing-inspect" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ec/c672e72dd66b633fb65e6011bc7c30c128e090364f8497660b7869e10123/apache_hamilton-1.90.0-py3-none-any.whl", hash = "sha256:95e356f7cb23e1a10d44fd0cf5efd3fb36a077b4ccb673950c92005bfdec2ff7", size = 475199, upload-time = "2026-04-25T18:45:17.514Z" }, +] + [[package]] name = "apache-hamilton-ui" version = "0.0.17" @@ -193,6 +209,7 @@ source = { editable = "." } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, + { name = "apache-hamilton" }, { name = "click" }, { name = "colorlog" }, { name = "ddtrace" }, @@ -204,7 +221,6 @@ dependencies = [ { name = "psycopg2-binary" }, { name = "pydantic" }, { name = "requests" }, - { name = "sf-hamilton" }, { name = "typer" }, ] @@ -212,6 +228,7 @@ dependencies = [ requires-dist = [ { name = "aiofiles" }, { name = "aiohttp", specifier = ">=3.13.0" }, + { name = "apache-hamilton", specifier = ">=1.90.0" }, { name = "click" }, { name = "colorlog" }, { name = "ddtrace", specifier = ">=2.21.0" }, @@ -223,7 +240,6 @@ requires-dist = [ { name = "psycopg2-binary" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "requests", specifier = ">=2.32.0" }, - { name = "sf-hamilton", specifier = ">=1.58.0" }, { name = "typer" }, ] @@ -1512,23 +1528,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, ] -[[package]] -name = "sf-hamilton" -version = "1.89.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions" }, - { name = "typing-inspect" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/12/1c/5205e36ca4eebfef29a6f75398812bfb923c9aad71fd9ad3248b7703a9b9/sf_hamilton-1.89.0.tar.gz", hash = "sha256:dec10455ce5dacd9402c7c8325c2eb2959c0f96f912dfcf2c999ef556421d2d2", size = 360574, upload-time = "2025-10-11T17:43:55.68Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/ce/3591411d90bf0f6829fec4403f77a63280c8eee942f55715ae1ed7ff8492/sf_hamilton-1.89.0-py3-none-any.whl", hash = "sha256:25c0efeb498cd1847aec5fbde2e69b4756c4e0c2a5238bbedbe2178d559fd677", size = 468919, upload-time = "2025-10-11T17:43:53.902Z" }, -] - [[package]] name = "shellingham" version = "1.5.4"