CSV + Steampipe

select
  first_name,
  last_name
from
  users;

+------------+-----------+
| first_name | last_name |
+------------+-----------+
| Michael    | Scott     |
| Dwight     | Schrute   |
+------------+-----------+

steampipe plugin install csv

connection "csv" {
  plugin = "csv"

  # Paths is a list of locations to search for CSV files
  # Paths can be configured with a local directory, a remote Git repository URL, or an S3 bucket URL
  # Refer https://hub.steampipe.io/plugins/turbot/csv#supported-path-formats for more information
  # All paths are resolved relative to the current working directory (CWD)
  # Wildcard based searches are supported, including recursive searches

  # For example:
  #  - "*.csv" matches all CSV files in the CWD
  #  - "*.csv.gz" matches all gzipped CSV files in the CWD
  #  - "**/*.csv" matches all CSV files in the CWD and all sub-directories
  #  - "../*.csv" matches all CSV files in the CWD's parent directory
  #  - "steampipe*.csv" matches all CSV files starting with "steampipe" in the CWD
  #  - "/path/to/dir/*.csv" matches all CSV files in a specific directory
  #  - "/path/to/dir/custom.csv" matches a specific file

  # If paths includes "*", all files (including non-CSV files) in
  # the CWD will be matched, which may cause errors if incompatible file types exist

  # Defaults to CWD
  paths = [ "*.csv", "*.csv.gz" ]

  # The field delimiter character when parsing CSV files. Must be a single
  # character. Defaults to comma.
  # separator = ","

  # If set, then lines beginning with the comment character without preceding
  # whitespace are ignored. Disabled by default.
  # comment = "#"

  # Determine whether to use the first row as the header row when creating column names.
  # Valid values are "auto", "on", "off":
  #   - "auto": If there are no empty or duplicate values use the first row as the header; else, use the first row as a data row and use generic column names, e.g., "a", "b".
  #   - "on": Use the first row as the header. If there are empty or duplicate values, the tables will fail to load.
  #   - "off": Do not use the first row as the header. All column names will be generic.
  # Defaults to "auto".
  # header = "auto"
}

connection "csv" {
  plugin = "csv"

  paths = [
    "*.csv",
    "**/*.csv",
    "github.com/codeforamerica/ohana-api//data/sample-csv//*.csv",
    "bitbucket.org/ved_misra/sample-csv//*.csv",
    "gitlab.com/subhajit7/example-files//sample-csv-files//*.csv",
    "s3::https://bucket.s3.us-east-1.amazonaws.com/test_folder//*.csv"
  ]
}

connection "csv" {
  plugin = "csv"

  paths = [ "*.csv", "~/*.csv", "/path/to/dir/custom.csv" ]
}

connection "csv" {
  plugin = "csv"

  paths = [ "github.com/codeforamerica/ohana-api//data/sample-csv//*.csv" ]
}

connection "csv" {
  plugin = "csv"

  paths = [
    "github.com/codeforamerica/ohana-api//data/sample-csv//*.csv",
    "bitbucket.org/ved_misra/sample-csv//*.csv",
    "gitlab.com/subhajit7/example-files//sample-csv-files//*.csv"
  ]
}

connection "csv" {
  plugin = "csv"

  paths = [
    "s3::https://bucket-2.s3.us-east-1.amazonaws.com//*.csv?aws_profile=<AWS_PROFILE>",
    "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//*.csv?aws_profile=<AWS_PROFILE>"
  ]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "ReadBucketObject",
      "Effect": "Allow",
      "Principal": {
        "AWS": "arn:aws:iam::123456789012:user/YOUR_USER"
      },
      "Action": ["s3:ListBucket", "s3:GetObject", "s3:GetObjectVersion"],
      "Resource": ["arn:aws:s3:::test-bucket1", "arn:aws:s3:::test-bucket1/*"]
    }
  ]
}

connection "config" {
  plugin = "config"

  paths = [
    "s3::https://bucket-1.s3.us-east-1.amazonaws.com/test_folder//*.csv",
    "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//**/*.csv"
  ]
}

/bin/sh -c "$(curl -fsSL https://steampipe.io/install/postgres.sh)" -- csv

CREATE EXTENSION IF NOT EXISTS steampipe_postgres_csv;
CREATE SERVER steampipe_csv FOREIGN DATA WRAPPER steampipe_postgres_csv OPTIONS (config '<your_config>');
CREATE SCHEMA csv;
IMPORT FOREIGN SCHEMA csv FROM SERVER steampipe_csv INTO csv;

/bin/sh -c "$(curl -fsSL https://steampipe.io/install/sqlite.sh)" -- csv

$ sqlite3
sqlite> .load ./steampipe_sqlite_extension_csv.so
sqlite> select steampipe_configure_csv('<your_config>');

/bin/sh -c "$(curl -fsSL https://steampipe.io/install/export.sh)" -- csv

steampipe_export_csv --config '<your_config>' <table_name>