stream#Readable TypeScript Examples

async function appReady(stdout: Readable): Promise<void> {
  return new Promise<void>((resolve) => {
    stdout.pipe(
      new Writable({
        write(chunk, encoding, callback) {
          if (/ready - started server on/i.test(stripAnsi(String(chunk)))) {
            resolve();
          }
          callback();
        },
      })
    );
  });
}

/**
 * @param binary Buffer
 * returns readableInstanceStream Readable
 */
export function bufferToStream(binary: Buffer): Readable {
    const readableInstanceStream = new Readable({
        read() {
            this.push(binary)
            this.push(null)
        },
    })

    return readableInstanceStream
}

function getMusic(
  url: string,
  cache?: { id: string; name: string; path: string; md5?: string }
): Readable | void {
  try {
    const data = got(url, {
      isStream: true,
      timeout: { response: 8000 },
    });
    if (cache) {
      const { id, name, path, md5 } = cache;
      data.on("end", () => void MusicCache.put(id, name, path, md5));
    }
    return data;
  } catch (err) {
    logError(err);
  }
  return;
}

export function createRandomNodeReadable(totalSize: number, chunkSize = 1000): Readable {
  if (totalSize % chunkSize !== 0) {
    throw new Error(`totalSize ${totalSize} is not dividable without remainder by chunkSize ${chunkSize}`)
  }

  const stream = new Readable()
  // eslint-disable-next-line @typescript-eslint/no-empty-function
  stream._read = (): void => {}

  for (let i = 0; i < totalSize / chunkSize; i++) {
    stream.push(randomByteArray(chunkSize))
  }

  stream.push(null)

  return stream
}

// toStream will recreate a read stream out of the supplied array of objects. This is generally used to prepare
// data to be sent to the Data Source interface's ReceiveData function. Users should avoid using this when possible
// and instead work with the data origin's stream (eg. http requests, file reads, conversion streams). However, we
// recognize there are instances in which we are sending so little data, or reading from a source with no stream
// implementations and so have provided this utility function.
export function toStream(data: any[]): Readable {
    const buffer = [...data]; // don't manipulate the underlying data array
    return new Readable({
        read() {
            if (buffer.length === 0) this.push(null);
            else this.push(buffer.shift());
        },
        objectMode: true,
    });
}

export async function readStreamUntilEnd(stream: Readable): Promise<Buffer> {
    const chunks = []
    let length = 0

    while (stream.readable) {
        const c = await stream.read()
        if (c === null) break

        length += c.length
        if (length > 2097152000) {
            throw new Error('File is too big')
        }

        chunks.push(c)
    }

    return Buffer.concat(chunks)
}

/**
     * Construct a IPC proxy
     * @param proc the process to wrap
     * @param acceptCommandTypes set of accepted commands for this process
     */
    constructor(proc: ChildProcess, acceptCommandTypes: Set<string>) {
        this.acceptCommandTypes = acceptCommandTypes;
        this.outgoingStream = <Writable>proc.stdio[ipcOutgoingFd];
        this.incomingStream = <Readable>proc.stdio[ipcIncomingFd];
        this.eventEmitter = new EventEmitter();
        this.readBuffer = Buffer.alloc(0);

        this.incomingStream.on('data', (data: Buffer) => { this.receive(data); });
        this.incomingStream.on('error', (error: Error) => { this.eventEmitter.emit('error', error); });
        this.outgoingStream.on('error', (error: Error) => { this.eventEmitter.emit('error', error); });
    }

/**
 * Creates a MavLink packet stream reader that is reading packets from the given input
 *
 * @param input input stream to read from
 */
export function createMavLinkStream(input: Readable, onCrcError: BufferCallback) {
  return input
    .pipe(new MavLinkPacketSplitter({}, onCrcError))
    .pipe(new MavLinkPacketParser())
}

describe("stream tests", function () {
  it("test", async function () {
    const signer = new ArweaveSigner(wallet0);
    const item = createData("hello", signer);
    const bundle = await bundleAndSignData([item], signer);

    const stream = Readable.from(bundle.getRaw());
    for await (const item of await processStream(stream)) {
      console.log(item);
    }
  });
});

constructor (private parser: parser) {
        super();
        parser.reset();

        this.readlineInput = new Readable();
        // https://stackoverflow.com/a/22085851
        // eslint-disable-next-line @typescript-eslint/no-empty-function
        this.readlineInput._read = () => {};

        this.readline = readline.createInterface({
            input: this.readlineInput,
            terminal: false,
        });
        this.readline.on("line", (line) => this.parser.feed_line(line));
        this.readline.on("close", () => {
            this.writable = false;
            this.emit("close");
            this.emit("finish");
        });
    }

retrieveFileContentStream(fileId: string): Promise<Readable> {
        return new Promise((resolve, reject) => {
            this.authenticator.authenticate().then((auth) => {
                const getParams = {
                    'fileId': fileId,
                    'alt': 'media'
                };
                const responseType = { responseType: 'stream' };
                const callbackFn = (err: any, response: any) => {
                    err ? reject(err) : resolve(response.data);
                };
                drive(auth).files.get(getParams, responseType, callbackFn);
            }).catch(err => reject(err));
        });
    }

/**
     * Custom POST request handler.
     * This is only needed to keep paths short in Windows – usually this module
     * is a dependency of a dependency and if it tries to require something
     * like the request module the paths get way too long to handle for Windows.
     * As we do only a simple POST request we do not actually require complicated
     * logic support (no redirects, no nothing) anyway.
     */
    postRequest(
        url: string,
        payload: string | Buffer | Readable | { [key: string]: string },
        params: XOAuth2.RequestParams,
        callback: (err: Error | null, buf: Buffer) => void
    ): void;

/**
     *
     * @param {Readable | string} stream
     * @param {{ inputType: StreamType, metadata: any|undefined }} options
     * @returns {AudioResource<Song>}
     */
    createAudioStream(stream: string | Readable , options: { inputType: StreamType, metadata?: any }): AudioResource<Song> {
        this.resource = createAudioResource(stream, {
            inputType: options.inputType,
            inlineVolume: true,
            metadata: options.metadata
        });

        return this.resource;
    }

async readTree(
    url: string,
    options?: ReadTreeOptions,
  ): Promise<ReadTreeResponse> {
    const { filepath } = parseGitUrl(url);

    const lastCommitShortHash = await this.getLastCommitShortHash(url);
    if (options?.etag && options.etag === lastCommitShortHash) {
      throw new NotModifiedError();
    }

    const downloadUrl = await getBitbucketCloudDownloadUrl(
      url,
      this.integration.config,
    );
    const archiveResponse = await fetch(
      downloadUrl,
      getBitbucketCloudRequestOptions(this.integration.config),
    );
    if (!archiveResponse.ok) {
      const message = `Failed to read tree from ${url}, ${archiveResponse.status} ${archiveResponse.statusText}`;
      if (archiveResponse.status === 404) {
        throw new NotFoundError(message);
      }
      throw new Error(message);
    }

    return await this.deps.treeResponseFactory.fromTarArchive({
      stream: archiveResponse.body as unknown as Readable,
      subpath: filepath,
      etag: lastCommitShortHash,
      filter: options?.filter,
    });
  }

private handleHubs(hubs: Hubs): void {
        this.log.debug("Got hubs:", hubs);
        const stationsSNs: string[] = Object.keys(this.stations);
        const newStationsSNs = Object.keys(hubs);
        for (const hub of Object.values(hubs)) {
            if (stationsSNs.includes(hub.station_sn)) {
                this.updateStation(hub);
            } else {
                const station = new Station(this.api, hub);
                station.on("connect", (station: Station) => this.onStationConnect(station));
                station.on("close", (station: Station) => this.onStationClose(station));
                station.on("raw device property changed", (deviceSN: string, params: RawValues) => this.updateDeviceProperties(deviceSN, params));
                station.on("livestream start", (station: Station, channel:number, metadata: StreamMetadata, videostream: Readable, audiostream: Readable) => this.onStartStationLivestream(station, channel, metadata, videostream, audiostream));
                station.on("livestream stop", (station: Station, channel:number) => this.onStopStationLivestream(station, channel));
                station.on("download start", (station: Station, channel: number, metadata: StreamMetadata, videoStream: Readable, audioStream: Readable) => this.onStationStartDownload(station, channel, metadata, videoStream, audioStream));
                station.on("download finish", (station: Station, channel: number) => this.onStationFinishDownload(station, channel));
                station.on("command result", (station: Station, result: CommandResult) => this.onStationCommandResult(station, result));
                station.on("guard mode", (station: Station, guardMode: number) => this.onStationGuardMode(station, guardMode));
                station.on("current mode", (station: Station, currentMode: number) => this.onStationCurrentMode(station, currentMode));
                station.on("rtsp livestream start", (station: Station, channel:number) => this.onStartStationRTSPLivestream(station, channel));
                station.on("rtsp livestream stop", (station: Station, channel:number) => this.onStopStationRTSPLivestream(station, channel));
                station.on("rtsp url", (station: Station, channel:number, value: string) => this.onStationRtspUrl(station, channel, value));
                station.on("property changed", (station: Station, name: string, value: PropertyValue) => this.onStationPropertyChanged(station, name, value));
                station.on("raw property changed", (station: Station, type: number, value: string) => this.onStationRawPropertyChanged(station, type, value));
                station.on("alarm event", (station: Station, alarmEvent: AlarmEvent) => this.onStationAlarmEvent(station, alarmEvent));
                station.on("runtime state", (station: Station, channel: number, batteryLevel: number, temperature: number) => this.onStationRuntimeState(station, channel, batteryLevel, temperature,));
                station.on("charging state", (station: Station, channel: number, chargeType: ChargingType, batteryLevel: number) => this.onStationChargingState(station, channel, chargeType, batteryLevel));
                station.on("wifi rssi", (station: Station, channel: number, rssi: number) => this.onStationWifiRssi(station, channel, rssi));
                station.on("floodlight manual switch", (station: Station, channel: number, enabled: boolean) => this.onFloodlightManualSwitch(station, channel, enabled));

                this.addStation(station);
            }
        }
        for (const stationSN of stationsSNs) {
            if (!newStationsSNs.includes(stationSN)) {
                this.removeStation(this.getStation(stationSN));
            }
        }
    }

async function main (): Promise<void> {
  let task: Listr<Ctx>

  logger.start('Example output from a task.')

  // eslint-disable-next-line prefer-const
  task = new Listr<Ctx>(
    [
      {
        title: 'This task will execute.',
        task: async (): Promise<Readable> => {
          return spawn('ls').stdout
        },
        options: {
          persistentOutput: true
        }
      }
    ],
    { concurrent: false }
  )

  try {
    const context = await task.run()

    logger.success(`Context: ${JSON.stringify(context)}`)
  } catch (e: any) {
    logger.fail(e)
  }
}

jest.mock('child_process', () => ({
  spawn: jest.fn().mockImplementationOnce(() => ({
    status: 0,
    stderr: Readable.from(['scrapy output']),
    stdout: Readable.from([
      `
line with no database output
database_output: {"body": "abc", "headers": [{"a": "b", "c": "d"}], "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "1de6816e1b0d07840082bed89f852b3f10688a5df6877a97460dbc474195d5dd", "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov/scans/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "6a2946030f804a281a0141397dbd948d7cae4698118bffd1c58e6d5f87480435", "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov/contributing/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "d8f190dfeaba948e31fc26e3ab7b7c774b1fbf1f6caac535e4837595eadf4795", "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov/usage/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"body": "abc", "headers": [{"a": "b", "c": "d"}], "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "226706ff585e907aa977323c404b9889f3b2e547d134060ed57fda2e2f1b9860", "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov/contributing/deployment/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "e1448fa789c02ddc90a37803150923359a4a21512e1737caec52be53ef3aa3b5", "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov/contributing/architecture/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "3091ca75bf2ee1e0bead7475adb2db8362f96b472d220fd0c29eaac639cdf37f", "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov/usage/customization/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
line with no database output {}
database_output: {"s3_key": "dd7e7e51a4a094e87ad88756842854c2d878ac55fb908035ddaf229c5568fa1a", "status": 200, "url": "https://docs.crossfeed.cyber.dhs.gov/usage/administration/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "afb6378d30bd1b43d86be5d1582d0e306ba6c9dd2c60f0dcbb1a3837b34cbe59", "status": 400, "url": "https://docs.crossfeed.cyber.dhs.gov/contributing/setup/", "domain_name": "docs.crossfeed.cyber.dhs.gov"}
database_output: {"s3_key": "afb6378d30bd1b43d86be5d1582d0e306ba6c9dd2c60f0dcbb1a3837b34cbe60", "status": 200, "url": "https://unrelated-url.cyber.dhs.gov/contributing/setup/", "domain_name": "unrelated-url.cyber.dhs.gov"}
`
    ])
  }))
}));

/**
   * Asynchronously executes the query and returns a {@link https://nodejs.org/api/stream.html#stream_class_stream_readable | Readable stream} that wraps the result set.
   * @param command - SQL command to execute
   * @param options - optional options object of type {@link IExecuteOptions | IExecuteOptions}
   *
   * @example
   * Streaming results of a DuckDB query into a CSV file:
   * ```ts
   * import { Connection, DuckDB, RowResultFormat } from "node-duckdb";
   * import { createWriteStream } from "fs";
   * import { Transform } from "stream";
   * class ArrayToCsvTransform extends Transform {
   *     constructor() {
   *        super({objectMode: true})
   *    }
   *    _transform(chunk: any[], _encoding: string, callback: any) {
   *        this.push(chunk.join(",") + '\n');
   *        callback();
   *    }
   * }
   *
   * async function outputToFileAsCsv() {
   *     const db = new DuckDB();
   *     const connection = new Connection(db);
   *     await connection.execute("CREATE TABLE people(id INTEGER, name VARCHAR);");
   *     await connection.execute("INSERT INTO people VALUES (1, 'Mark'), (2, 'Hannes'), (3, 'Bob');");
   *     const resultStream = await connection.execute("SELECT * FROM people;", {rowResultFormat: RowResultFormat.Array});
   *     const transformToCsvStream = new ArrayToCsvTransform();
   *     const writeStream = createWriteStream("my-people-output");
   *     resultStream.pipe(transformToCsvStream).pipe(writeStream);
   * }
   * outputToFileAsCsv();
   * ```
   */
  public async execute<T>(command: string, options?: IExecuteOptions): Promise<Readable> {
    const resultIteratorBinding = await this.connectionBinding.execute<T>(command, options);
    return getResultStream(new ResultIterator(resultIteratorBinding));
  }

async function getAllReadable(read: Readable): Promise<any[]> {
  const arr = []

  for await (const readElement of read) {
    arr.push(readElement)
  }

  return arr
}

async set(path: string, value: any) {
		path = getPath(path);
		fse.ensureDirSync(dirname(path));

		value = Readable.from(value);
		const cleaned_file = fs.createWriteStream(path);

		return value.pipe(new ExifTransformer()).pipe(cleaned_file);
	}

constructor () {
		super();
		this.stderr = new Readable({
			read () {},
		});
		this.stdout = new Readable({
			read () {},
		});
	}

public execute = async (
    options: Options,
    input: Readable,
  ): Promise<TesseractResult> => {
    const pool = await this._getPool(options);
    const instance = await pool.acquire();
    const result = await instance.execute(input);
    await pool.release(instance);
    return this._treatLineEndings(result);
  };

/*
        uploadFile should be used when uploading an actual file, not for manipulating
        a file record in storage. This should hopefully be obvious as the function
        signature requires a Readable stream
     */
    async uploadFile(
        containerID: string,
        dataSourceID: string,
        user: User,
        filename: string,
        encoding: string,
        mimetype: string,
        stream: Readable,
    ): Promise<Result<File>> {
        const provider = BlobStorageProvider();

        if (!provider) return Promise.resolve(Result.Failure('no storage provider set'));

        // run the actual file upload the storage provider
        const result = await provider.uploadPipe(`containers/${containerID}/datasources/${dataSourceID}/`, filename, stream);
        if (result.isError) return Promise.resolve(Result.Pass(result));

        const file = new File({
            file_name: filename,
            file_size: result.value.size,
            md5hash: result.value.md5hash,
            adapter_file_path: result.value.filepath,
            adapter: provider.name(),
            metadata: result.value.metadata,
            container_id: containerID,
            data_source_id: dataSourceID,
        });

        const saved = await this.save(file, user);
        if (saved.isError) return Promise.resolve(Result.Pass(saved));

        return Promise.resolve(Result.Success(file));
    }

export function bufferToStream(binary: Buffer) {

	const readableInstanceStream = new Readable({
		read() {
			this.push(binary)
			this.push(null)
		}
	})

	return readableInstanceStream
}

constructor(id: number) {
    super();

    this.id = id;

    this.body = new Readable();
    this.body._read = () => {
      return;
    };
  }

export async function createContainer(
  image: string,
  bin: string,
  env: string[],
  auth: DockerConfigJson,
  log: RunLogger,
  volumes: { from: string, to: string }[],
  workdir: string,
) {
  const pullCredentials = getCredentials(image, auth);
  const pullStream: Readable = await docker.pull(image, { authconfig: pullCredentials });
  await promisifyStream(pullStream, chunk => {
    const lines = chunk.toString().trim().split('\r\n');
    lines.forEach(line => {
      const { status, progress } = JSON.parse(line) as PullJson;
      log(`${status}${progress ? ` ${progress}` : ''}\n`, 'system');
    });
  });
  logger.debug('image pulled');

  // https://docs.docker.com/engine/api/v1.37/#operation/ContainerCreate
  const container = await docker.createContainer({
    Image: image,
    Env: env,
    Cmd: [bin],
    Tty: true,
    HostConfig: { Binds: volumes.map(item => `${item.from}:${item.to}`) },
    WorkingDir: workdir,
    AttachStdin: false,
    AttachStdout: false,
    AttachStderr: false,
  });
  logger.debug('container created');

  await container.start();
  logger.debug('container started');

  return container;
}

public async get(
        url: string,
        timeoutInMs: number,
        retries: number,
        retryDelayInMs: number,
        cancellationToken?: CancellationToken,
        onDownloadProgressChange?: (downloadedBytes: number, totalBytes: number) => void
    ): Promise<Readable> {
        const requestFn = () => this.getRequestHelper(
            url,
            timeoutInMs,
            cancellationToken,
            onDownloadProgressChange
        );
        const errorHandlerFn = (error: Error) => {
            const statusCode = (error as any)?.response?.status;
            if (statusCode != null && 400 <= statusCode && statusCode < 500) {
                throw error;
            }
        };
        return RetryUtility.exponentialRetryAsync(requestFn, `HttpRequestHandler.get`, retries, retryDelayInMs, errorHandlerFn);
    }

export async function streamToBuffer(str: Readable): Promise<Buffer> {
  return await new Promise((resolve, reject) => {
    let data: any[] = []
    str
      .on('error', reject)
      .on('data', (chunk) => data.push(chunk))
      .on('end', () => resolve(Buffer.concat(data)))
  })
}

/**
 * Download a file and return it as a Node readable stream,
 * streaming file contents.
 *
 * @param params  File download parameters
 * @internal
 */
export function downloadAsStream(
    this: TelegramClient,
    params: FileDownloadParameters
): Readable {
    if (
        params.location instanceof FileLocation &&
        Buffer.isBuffer(params.location.location)
    ) {
        return bufferToStream(params.location.location)
    }

    const ret = new Readable({
        async read() {},
    })

    setTimeout(async () => {
        try {
            for await (const chunk of this.downloadAsIterable(params)) {
                ret.push(chunk)
            }

            ret.push(null)
        } catch (e) {
            ret.emit('error', e)
        }
    }, 0)

    return ret
}