Fully managed Spark Clusters in your own AWS account — with no compute markup

class IcebergS3Example(SparklithEntryPoint):

    def run(self):
        bucket = "<your-bucket-name>"
        warehouse_path = f"s3a://{bucket}/iceberg-warehouse"

        with OrchesteraSparkSession(
            app_name="IcebergS3Example",
            executor_instances=4,
            executor_cores=2,
            executor_memory="8g",
        ) as spark:
            spark.sparkContext.setLogLevel("ERROR")

            df = spark.read.parquet(
                "s3a://ookla-open-data/parquet/performance/..."
            ).limit(1000)

            table_name = "local.example.ookla_performance"
            spark.sql("CREATE NAMESPACE IF NOT EXISTS local.example")
            df.writeTo(table_name).createOrReplace()

            iceberg_df = spark.table(table_name)
            spark.sql(f"SELECT * FROM {table_name}.history").show()