import { getToken } from "./get_auth_token.js";
import { getConnectorInformationsToCreateAConnection } from "./get_connectors_informations.js";

const createPipelineAwsS3 = async () => {
  const apiToken = await getToken();
  const connectorsInformations =
    await getConnectorInformationsToCreateAConnection();
  const awsS3Connector = connectorsInformations
    .filter((connector) => connector.plugin === "aws_s3")
    .pop();
  const connectionInformations = await createConnection(
    apiToken,
    awsS3Connector
  );
  return await createPipeline(apiToken, connectionInformations);
};

const createConnection = async (token, connectorInformations) => {
  const url = "https://maestro.dadosfera.ai/connections";
  const data = {
    name: "your-connection-name",
    description: "your-connection-description",
    properties: {
      credentials_type: "iam_user",
      aws_access_key_id: "your_aws_access_key_id",
      aws_secret_access_key: "your_aws_secret_access_key",
    },
    ...connectorInformations,
  };

  return fetch(url, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `${token}`,
    },
    body: JSON.stringify(data),
  })
    .then((response) => {
      if (response.ok) return response.json();
      else throw response.statusText;
    })
    .then((data) => {
      const [connector_plugin, connector_version] =
        data.connection.connector_id.split("-");

      return {
        connection_id: data.connection.id,
        connector_name: data.connection.connector_name,
        plugin: data.connection.plugin,
        connector_plugin,
        connector_version,
        connector_id: data.connection.connector_id,
        image_url: data.connection.image_url,
        type: data.connection.type,
      };
    });
};

const createPipeline = async (token, connectionInformations) => {
  const url = "https://maestro.dadosfera.ai/pipelinesV2";
  const data = {
    name: "your-pipeline-name",
    description: "your-pipeline-description",
    transformations_ids: [],
    tags: [],
    cron: "@once", // schedule with cron https://crontab.guru/
    config: {
      cron: "@once", // schedule with cron https://crontab.guru/
      tables: [],
    },

    properties: {
      credentials_type: "iam_user",
      source_bucket: "bucket-name",
      source_prefix: "fil/to/be/extracted",

      engine: "csv",
      file_format_params: {
        encoding: "UTF-8",
        sep: ",",
        header: true,
      },

      //   engine: "json",
      //   file_format_params: {
      //     encoding: "UTF-8"
      //   },

      //   engine: "parquet",
      //   file_format_params: {},
    },

    ...connectionInformations,
  };

  return fetch(url, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `${token}`,
    },
    body: JSON.stringify(data),
  })
    .then((response) => {
      if (response.ok) return response.json();
      else throw response.statusText;
    })
    .then((data) => {
      return {
        id: data.pipeline.id,
        name: data.pipeline.name,
        created_at: data.pipeline.created_at,
        status: data.pipeline.status,
        connector_plugin: data.pipeline.connector_plugin,
        connection_id: data.pipeline.connection_id,
        type: data.pipeline.type,
      };
    });
};

{"success":true}

Get API auth token

For any API request it is necessary to have the authentication token, this recipe demonstrates how to get it.

Get essential information from connectors

To create a pipeline connection, you need some information about the connector to be used, so this recipe will help with that step.

Create a connection

To create a connection it is necessary to pass the connector information and set some parameters.

Configure create connection response

In this step, the base information of the connection is defined to be used in the creation of the pipeline.

Create a pipeline

To create a pipeline it is necessary to pass the connection information and set some parameters.

Configure create pipeline response

In this step, the base information of the pipeline is defined to be used in the function response.

Observation

You can see that the aws s3 pipeline has three different types of files that can be collected, so you must use the correct configuration for a given type of file.