Added simple har builder (#2)

* Added simple har builder

* Updating readme

* Capturing body information

* Removing debugger call

Author: kevincai3

README.md

@@ -15,3 +15,5 @@ To setup:
 `prettier` is used with `husky` and `lint-staged` to lint on commit, which should be installed automatically by `npm`.
 
 Use `yarn test` to run the test suite.
+
+Can also use `yarn test-with-debugger` to run the builtin node debugger by placing a `debugger;` anywhere in the code base.

jest.config.ts

@@ -2,4 +2,5 @@
 module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
+  testPathIgnorePatterns: ["dist"],
 };

package.json

@@ -6,11 +6,15 @@
   "author": "",
   "license": "MIT",
   "devDependencies": {
+    "@types/content-type": "^1.1.5",
+    "@types/cookie": "^0.5.1",
     "@types/express": "^4.17.13",
     "@types/har-format": "^1.2.8",
     "@types/jest": "^28.1.6",
     "@types/node": "^18.6.0",
+    "@types/set-cookie-parser": "^2.4.2",
     "@types/supertest": "^2.0.12",
+    "express": "^4.16.0",
     "husky": "^8.0.1",
     "jest": "^28.1.3",
     "lint-staged": "^13.0.3",
@@ -21,14 +25,20 @@
     "typescript": "^4.7.4"
   },
   "peerDepedencies": {
-    "express": "^4.18.1"
+    "express": "^4.16.0"
   },
   "scripts": {
     "build": "yarn tsc",
     "test": "yarn jest",
+    "test-with-debugger": "node inspect ./node_modules/.bin/jest --runInBand",
     "prepare": "yarn husky install"
   },
-  "dependencies": {},
+  "dependencies": {
+    "content-type": "^1.0.4",
+    "cookie": "^0.5.0",
+    "raw-body": "^2.5.1",
+    "set-cookie-parser": "^2.5.1"
+  },
   "engines": {
     "node": ">=16.0.0"
   },

src/format/har.ts

@@ -0,0 +1,265 @@
+import url from "url";
+import http from "http";
+
+import cookie from "cookie";
+import setCookie from "set-cookie-parser";
+import {
+  Har as HarType,
+  Cookie,
+  Entry,
+  Request as HarRequest,
+  Response as HarResponse,
+  Header,
+  QueryString,
+  PostData,
+  Content,
+} from "har-format";
+
+// Cannot get TS to point to node's types, it's defaulting to types from the fetchApi
+type Request = any;
+type Response = any;
+
+export type AdditionalData = {
+  reqBodyString?: string;
+  reqBodyStringComment?: string;
+  responseText?: string;
+  responseSize?: number;
+};
+
+export class Har {
+  har: HarType;
+
+  public constructor() {
+    this.har = {
+      log: {
+        version: "1.2",
+        creator: {
+          name: "Speakeasy Typescript SDK",
+          version: "0.0.1", //TODO(kevinc): Use actual version
+        },
+        browser: {
+          name: "",
+          version: "",
+        },
+        pages: [],
+        entries: [],
+        comment: "",
+      },
+    };
+  }
+
+  public populate(req: Request, res: Response, additionalData: AdditionalData) {
+    this.har.log.entries.push(this.buildEntry(req, res, additionalData));
+  }
+
+  public static buildHar(
+    req: Request,
+    res: Response,
+    additionalData: AdditionalData = {}
+  ): Har {
+    const har = new Har();
+    har.populate(req, res, additionalData);
+    return har;
+  }
+
+  public toString(): string {
+    return JSON.stringify(this.har);
+  }
+
+  private buildEntry(
+    req: Request,
+    res: Response,
+    additionalData: AdditionalData
+  ): Entry {
+    return {
+      pageref: "page_0",
+      startedDateTime: new Date().toISOString(), //TODO(kevinc): Get the actual start date time from the header
+      time: 0, //TODO(kevinc): Compute the actual elapsed time
+      request: this.buildRequest(req, additionalData),
+      response: this.buildResponse(res, additionalData),
+      cache: {},
+      timings: {
+        send: 0,
+        receive: 0,
+        wait: 0,
+      },
+    };
+  }
+
+  private buildRequest(
+    req: Request,
+    additionalData: AdditionalData
+  ): HarRequest {
+    const result: HarRequest = {
+      method: req.method,
+      url: req.url,
+      httpVersion: req.httpVersion,
+      cookies: this.buildRequestCookie(req),
+      headers: this.buildRequestHeaders(req.headers),
+      queryString: this.buildQueryString(req.url),
+      headersSize: this.buildRequestHeadersSize(req),
+      bodySize: -1,
+    };
+    const postData = this.buildRequestPostData(req, additionalData);
+    if (postData != null) {
+      result.postData = postData;
+      result.bodySize = postData.text?.length ?? -1;
+    }
+
+    return result;
+  }
+
+  private buildResponse(
+    res: Response,
+    additionalData: AdditionalData
+  ): HarResponse {
+    const content = this.buildResponseContent(res, additionalData);
+    const result: HarResponse = {
+      status: res.statusCode,
+      statusText: res.statusMessage,
+      httpVersion: "", // TODO(kevinc): Response HTTP version
+      cookies: this.buildResponseCookie(res),
+      headers: this.buildResponseHeaders(res.getHeaders()),
+      content: content,
+      redirectURL: res.getHeader("location") ?? "",
+      headersSize: this.buildResponseHeadersSize(res.req, res),
+      bodySize: content.size,
+    };
+
+    return result;
+  }
+
+  private buildRequestCookie(req: Request): Cookie[] {
+    const cookies = req.header("cookie");
+    const entries: Cookie[] = [];
+    if (cookies == null) {
+      return [];
+    }
+    Object.entries(cookie.parse(cookies)).forEach(([cookieName, cookieVal]) => {
+      entries.push({
+        name: cookieName,
+        value: cookieVal,
+      });
+    });
+    return entries;
+  }
+
+  private buildRequestPostData(
+    req: Request,
+    additionalData: AdditionalData
+  ): PostData | undefined {
+    if (additionalData.reqBodyString == undefined) {
+      return undefined;
+    }
+    return {
+      // We don't parse the body params
+      mimeType: req.getHeader("content-type"),
+      text: additionalData.reqBodyString,
+    };
+  }
+
+  private buildResponseCookie(res: Response): Cookie[] {
+    return setCookie.parse(res).map((cookie) => ({
+      name: cookie.name,
+      value: cookie.value,
+      path: cookie.path,
+      domain: cookie.domain,
+      expires:
+        cookie.expires == null ? undefined : cookie.expires.toUTCString(),
+      httpOnly: cookie.httpOnly,
+      secure: cookie.secure,
+    }));
+  }
+
+  private buildRequestHeaders(headers: Object): Header[] {
+    return Object.entries(headers).map(([name, value]) => ({
+      name,
+      value,
+    }));
+  }
+
+  private buildRequestHeadersSize(req: Request): number {
+    // Building string to get actual byte size
+    let rawHeaders = "";
+    rawHeaders += `${req.method} ${req.url} HTTP/${req.httpVersion}\r\n`;
+    for (let i = 0; i < req.rawHeaders.length; i += 2) {
+      rawHeaders += `${req.rawHeaders[i]}: ${req.rawHeaders[i + 1]}\r\n`;
+    }
+    rawHeaders += "\r\n";
+    return Buffer.byteLength(rawHeaders, "utf8");
+  }
+
+  private buildResponseHeadersSize(req: Request, res: Response): number {
+    // Building string to get actual byte size
+    let rawHeaders = "";
+    rawHeaders += `HTTP/${req.httpVersion} ${res.statusCode} ${res.statusMessage}\r\n`;
+    Object.entries(res.getHeaders()).forEach(([header, value]) => {
+      rawHeaders += `${header}: ${value}`;
+    });
+    rawHeaders += "\r\n";
+    return Buffer.byteLength(rawHeaders, "utf8");
+  }
+
+  private buildResponseHeaders(headers: Object): Header[] {
+    const responseHeaders: Header[] = [];
+    Object.entries(headers).forEach(([name, value]) => {
+      if (Array.isArray(value)) {
+        responseHeaders.concat(
+          value.map((val) => ({
+            name: name,
+            value: val,
+          }))
+        );
+      } else {
+        responseHeaders.push({
+          name,
+          value,
+        });
+      }
+    });
+    return responseHeaders;
+  }
+
+  private buildResponseContent(
+    res: Response,
+    additionalData: AdditionalData
+  ): Content {
+    const content: Content = {
+      size: -1,
+      mimeType: res.getHeaders()["content-type"] ?? "",
+    };
+    if (additionalData.responseText != null) {
+      content.text = additionalData.responseText;
+    }
+    if (additionalData.responseSize != null) {
+      content.size = additionalData.responseSize;
+    }
+    return content;
+  }
+
+  private buildQueryString(urlStr: string): QueryString[] {
+    const queryObj = url.parse(urlStr, true).query;
+    const queryString: QueryString[] = [];
+    Object.entries(queryObj).forEach(([name, value]) => {
+      if (value === undefined) {
+        queryString.push({
+          name: name,
+          value: "",
+        });
+      } else if (Array.isArray(value)) {
+        queryString.concat(
+          value.map((val) => ({
+            name: name,
+            value: val,
+          }))
+        );
+      } else {
+        queryString.push({
+          name,
+          value,
+        });
+      }
+    });
+    return queryString;
+  }
+}

src/middleware/express/middleware.ts

@@ -1,7 +1,13 @@
+import { Duplex } from "stream";
+
 import { Request, Response } from "express";
+import getRawBody from "raw-body";
+import { parse as parseContentType } from "content-type";
+
 import { getInstance } from "../../speakeasy";
 import { sendApiCall } from "../../transport";
 import { outputError } from "../../error";
+import { Har, AdditionalData } from "../../format/har";
 
 export function expressMiddleware() {
   return function (req: Request, res: Response, next: () => void) {
@@ -11,8 +17,64 @@ export function expressMiddleware() {
       return next();
     }
 
+    // Capturing request body
+    let rawBody: null | string = null;
+    let rawBodyParserErr: null | Error = null;
+    const rawBodyOptions: {
+      encoding: string;
+    } = {
+      encoding: "utf-8",
+    };
+    try {
+      rawBodyOptions.encoding = parseContentType(req).parameters.charset;
+    } catch (err) {} // We don't decode if we can't parse the charset
+    getRawBody(req, rawBodyOptions, (err, result) => {
+      if (err) {
+        rawBodyParserErr = err;
+        return;
+        // We skip logging the body if there's an error
+      }
+      rawBody = result;
+    });
+
+    // Capturing response body by overwriting the default write and end methods
+    const oldWrite = res.write;
+    const oldEnd = res.end;
+
+    const responseChunks: any[] = [];
+
+    res.write = (chunk, ...args) => {
+      responseChunks.push(chunk);
+      //@ts-ignore TS cannot type args correctly
+      return oldWrite.apply(res, [chunk, ...args]);
+    };
+
+    //@ts-ignore TS cannot type args correctly
+    res.end = (...args) => {
+      // #end has 3 signatures, the first argument is a chunk only if there are more than 1 argument
+      if (args.length > 1) {
+        responseChunks.push(args[0]);
+      }
+      //@ts-ignore TS cannot type args correctly
+      return oldEnd.apply(res, args);
+    };
+
     res.on("finish", () => {
-      sendApiCall();
+      const additionalData: AdditionalData = {};
+      if (rawBody != null) {
+        additionalData.reqBodyString = rawBody;
+      }
+      if (rawBodyParserErr != null) {
+        additionalData.reqBodyStringComment = `Failed to capture body string: ${rawBodyParserErr.message}`;
+      }
+      if (responseChunks.length > 0) {
+        const responseBuffer = Buffer.concat(responseChunks);
+        additionalData.responseText = responseBuffer.toString("utf8");
+        additionalData.responseSize = responseBuffer.byteLength;
+      }
+
+      const har = Har.buildHar(req, res, additionalData);
+      sendApiCall(har);
     });
     next();
   };