Fix/office365 api call (#3534)

* WIP fixes for handling office365 api call * First working version microsoft api call handling * Remove logs * Clean up and improve function names * Clean up * Remove log * Fix function missing its correct name * Fix direct return of statement * Cleanup * Simplify with private fetcher * Update CalendarService.ts * Update getOfficeAppKeys.ts Co-authored-by: zomars <zomars@me.com> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
2022-07-27 19:12:42 +00:00
parent 7537ea5b91
commit 525ce0d075
9 changed files with 271 additions and 109 deletions
@@ -1,4 +1,4 @@
-import { Prisma } from "@prisma/client";
+import type { Prisma } from "@prisma/client";

 import prisma from "@calcom/prisma";

@@ -0,0 +1,10 @@
+import type Zod from "zod";
+
+import getAppKeysFromSlug from "./getAppKeysFromSlug";
+
+export async function getParsedAppKeysFromSlug(slug: string, schema: Zod.Schema) {
+  const appKeys = await getAppKeysFromSlug(slug);
+  return schema.parse(appKeys);
+}
+
+export default getParsedAppKeysFromSlug;
@@ -3,7 +3,6 @@ import { Credential } from "@prisma/client";

 import { getLocation, getRichDescription } from "@calcom/lib/CalEventParser";
 import { handleErrorsJson, handleErrorsRaw } from "@calcom/lib/errors";
-import { HttpError } from "@calcom/lib/http-error";
 import logger from "@calcom/lib/logger";
 import prisma from "@calcom/prisma";
 import type { BufferedBusyTime } from "@calcom/types/BufferedBusyTime";
@@ -15,39 +14,52 @@ import type {
  NewCalendarEventType,
 } from "@calcom/types/Calendar";

-import getAppKeysFromSlug from "../../_utils/getAppKeysFromSlug";
 import { O365AuthCredentials } from "../types/Office365Calendar";
+import { getOfficeAppKeys } from "./getOfficeAppKeys";

-let client_id = "";
-let client_secret = "";
+interface IRequest {
+  method: string;
+  url: string;
+  id: number;
+}
+
+interface ISettledResponse {
+  id: string;
+  status: number;
+  headers: {
+    "Retry-After": string;
+    "Content-Type": string;
+  };
+  body: Record<string, any>;
+}
+
+interface IBatchResponse {
+  responses: ISettledResponse[];
+}

 export default class Office365CalendarService implements Calendar {
  private url = "";
  private integrationName = "";
  private log: typeof logger;
-  auth: Promise<{ getToken: () => Promise<string> }>;
+  private accessToken: string | null = null;
+  auth: { getToken: () => Promise<string> };
+  private apiGraphUrl = "https://graph.microsoft.com/v1.0";

  constructor(credential: Credential) {
    this.integrationName = "office365_calendar";
-    this.auth = this.o365Auth(credential).then((t) => t);
+    this.auth = this.o365Auth(credential);

    this.log = logger.getChildLogger({ prefix: [`[[lib] ${this.integrationName}`] });
  }

  async createEvent(event: CalendarEvent): Promise<NewCalendarEventType> {
    try {
-      const accessToken = await (await this.auth).getToken();
-
      const calendarId = event.destinationCalendar?.externalId
        ? `${event.destinationCalendar.externalId}/`
        : "";

-      const response = await fetch(`https://graph.microsoft.com/v1.0/me/calendars/${calendarId}events`, {
+      const response = await this.fetcher(`/me/calendars/${calendarId}events`, {
        method: "POST",
-        headers: {
-          Authorization: "Bearer " + accessToken,
-          "Content-Type": "application/json",
-        },
        body: JSON.stringify(this.translateEvent(event)),
      });

@@ -61,14 +73,8 @@ export default class Office365CalendarService implements Calendar {

  async updateEvent(uid: string, event: CalendarEvent): Promise<any> {
    try {
-      const accessToken = await (await this.auth).getToken();
-
-      const response = await fetch("https://graph.microsoft.com/v1.0/me/calendar/events/" + uid, {
+      const response = await this.fetcher(`/me/calendar/events/${uid}`, {
        method: "PATCH",
-        headers: {
-          Authorization: "Bearer " + accessToken,
-          "Content-Type": "application/json",
-        },
        body: JSON.stringify(this.translateEvent(event)),
      });

@@ -82,13 +88,8 @@ export default class Office365CalendarService implements Calendar {

  async deleteEvent(uid: string): Promise<void> {
    try {
-      const accessToken = await (await this.auth).getToken();
-
-      const response = await fetch("https://graph.microsoft.com/v1.0/me/calendar/events/" + uid, {
+      const response = await this.fetcher(`/me/calendar/events/${uid}`, {
        method: "DELETE",
-        headers: {
-          Authorization: "Bearer " + accessToken,
-        },
      });

      handleErrorsRaw(response);
@@ -110,95 +111,72 @@ export default class Office365CalendarService implements Calendar {
    const filter = `?startDateTime=${encodeURIComponent(
      dateFromParsed.toISOString()
    )}&endDateTime=${encodeURIComponent(dateToParsed.toISOString())}`;
-    return (await this.auth)
-      .getToken()
-      .then(async (accessToken) => {
-        const selectedCalendarIds = selectedCalendars
-          .filter((e) => e.integration === this.integrationName)
-          .map((e) => e.externalId)
-          .filter(Boolean);
-        if (selectedCalendarIds.length === 0 && selectedCalendars.length > 0) {
-          // Only calendars of other integrations selected
-          return Promise.resolve([]);
-        }

-        const ids = await (selectedCalendarIds.length === 0
-          ? this.listCalendars().then((cals) => cals.map((e_2) => e_2.externalId).filter(Boolean) || [])
-          : Promise.resolve(selectedCalendarIds));
-        const requests = ids.map((calendarId, id) => ({
-          id,
-          method: "GET",
-          url: `/me/calendars/${calendarId}/calendarView${filter}`,
-        }));
-        const response = await fetch("https://graph.microsoft.com/v1.0/$batch", {
-          method: "POST",
-          headers: {
-            Authorization: "Bearer " + accessToken,
-            "Content-Type": "application/json",
-          },
-          body: JSON.stringify({ requests }),
-        });
-        const responseBody = await handleErrorsJson(response);
-        return responseBody.responses.reduce(
-          (acc: BufferedBusyTime[], subResponse: { body: { value?: any[] } }) =>
-            acc.concat(
-              subResponse.body?.value
-                ? subResponse.body.value
-                    .filter((evt) => evt.showAs !== "free" && evt.showAs !== "workingElsewhere")
-                    .map((evt) => {
-                      return {
-                        start: evt.start.dateTime + "Z",
-                        end: evt.end.dateTime + "Z",
-                      };
-                    })
-                : []
-            ),
-          []
-        );
-      })
-      .catch((err: unknown) => {
-        console.log(err);
-        return Promise.reject([]);
-      });
+    try {
+      const selectedCalendarIds = selectedCalendars
+        .filter((e) => e.integration === this.integrationName)
+        .map((e) => e.externalId)
+        .filter(Boolean);
+      if (selectedCalendarIds.length === 0 && selectedCalendars.length > 0) {
+        // Only calendars of other integrations selected
+        return Promise.resolve([]);
+      }
+
+      const ids = await (selectedCalendarIds.length === 0
+        ? this.listCalendars().then((cals) => cals.map((e_2) => e_2.externalId).filter(Boolean) || [])
+        : Promise.resolve(selectedCalendarIds));
+      const requests = ids.map((calendarId, id) => ({
+        id,
+        method: "GET",
+        url: `/me/calendars/${calendarId}/calendarView${filter}`,
+      }));
+      const response = await this.apiGraphBatchCall(requests);
+      const responseBody = await handleErrorsJson(response);
+      let responseBatchApi: IBatchResponse = { responses: [] };
+      if (typeof responseBody === "string") {
+        responseBatchApi = this.handleTextJsonResponseWithHtmlInBody(responseBody);
+      }
+      let alreadySuccessResponse = [] as ISettledResponse[];
+
+      // Validate if any 429 status Retry-After is present
+      const retryAfter =
+        !!responseBatchApi?.responses && this.findRetryAfterResponse(responseBatchApi.responses);
+
+      if (retryAfter && responseBatchApi.responses) {
+        responseBatchApi = await this.fetchRequestWithRetryAfter(requests, responseBatchApi.responses, 2);
+      }
+
+      // Recursively fetch nextLink responses
+      alreadySuccessResponse = await this.fetchResponsesWithNextLink(responseBatchApi.responses);
+
+      return alreadySuccessResponse ? this.processBusyTimes(alreadySuccessResponse) : [];
+    } catch (err) {
+      console.log(err);
+      return Promise.reject([]);
+    }
  }

  async listCalendars(): Promise<IntegrationCalendar[]> {
-    return (await this.auth).getToken().then((accessToken) =>
-      fetch("https://graph.microsoft.com/v1.0/me/calendars", {
-        method: "get",
-        headers: {
-          Authorization: "Bearer " + accessToken,
-          "Content-Type": "application/json",
-        },
-      })
-        .then(handleErrorsJson)
-        .then((responseBody: { value: OfficeCalendar[] }) => {
-          return responseBody.value.map((cal) => {
-            const calendar: IntegrationCalendar = {
-              externalId: cal.id ?? "No Id",
-              integration: this.integrationName,
-              name: cal.name ?? "No calendar name",
-              primary: cal.isDefaultCalendar ?? false,
-              readOnly: !cal.canEdit && true,
-            };
-            return calendar;
-          });
-        })
-    );
+    const response = await this.fetcher(`/me/calendars`);
+    const responseBody = (await handleErrorsJson(response)) as { value: OfficeCalendar[] };
+    return responseBody.value.map((cal) => {
+      const calendar: IntegrationCalendar = {
+        externalId: cal.id ?? "No Id",
+        integration: this.integrationName,
+        name: cal.name ?? "No calendar name",
+        primary: cal.isDefaultCalendar ?? false,
+        readOnly: !cal.canEdit && true,
+      };
+      return calendar;
+    });
  }

-  private o365Auth = async (credential: Credential) => {
-    const appKeys = await getAppKeysFromSlug("office365-calendar");
-    if (typeof appKeys.client_id === "string") client_id = appKeys.client_id;
-    if (typeof appKeys.client_secret === "string") client_secret = appKeys.client_secret;
-    if (!client_id) throw new HttpError({ statusCode: 400, message: "office365 client_id missing." });
-    if (!client_secret) throw new HttpError({ statusCode: 400, message: "office365 client_secret missing." });
-
+  private o365Auth = (credential: Credential) => {
    const isExpired = (expiryDate: number) => expiryDate < Math.round(+new Date() / 1000);
-
    const o365AuthCredentials = credential.key as O365AuthCredentials;

    const refreshAccessToken = async (refreshToken: string) => {
+      const { client_id, client_secret } = await getOfficeAppKeys();
      const response = await fetch("https://login.microsoftonline.com/common/oauth2/v2.0/token", {
        method: "POST",
        headers: { "Content-Type": "application/x-www-form-urlencoded" },
@@ -257,4 +235,158 @@ export default class Office365CalendarService implements Calendar {
      location: event.location ? { displayName: getLocation(event) } : undefined,
    };
  };
+
+  private fetcher = async (endpoint: string, init?: RequestInit | undefined) => {
+    this.accessToken = await this.auth.getToken();
+    return fetch(`${this.apiGraphUrl}${endpoint}`, {
+      method: "get",
+      headers: {
+        Authorization: "Bearer " + this.accessToken,
+        "Content-Type": "application/json",
+      },
+      ...init,
+    });
+  };
+
+  private fetchResponsesWithNextLink = async (
+    settledResponses: ISettledResponse[]
+  ): Promise<ISettledResponse[]> => {
+    const alreadySuccess = [] as ISettledResponse[];
+    const newLinkRequest = [] as IRequest[];
+    settledResponses?.forEach((response) => {
+      if (response.status === 200 && response.body["@odata.nextLink"] === undefined) {
+        alreadySuccess.push(response);
+      } else {
+        const nextLinkUrl = response.body["@odata.nextLink"]
+          ? response.body["@odata.nextLink"].replace(this.apiGraphUrl, "")
+          : "";
+        if (nextLinkUrl) {
+          // Saving link for later use
+          newLinkRequest.push({
+            id: Number(response.id),
+            method: "GET",
+            url: nextLinkUrl,
+          });
+        }
+        delete response.body["@odata.nextLink"];
+        // Pushing success body content
+        alreadySuccess.push(response);
+      }
+    });
+
+    if (newLinkRequest.length === 0) {
+      return alreadySuccess;
+    }
+
+    const newResponse = await this.apiGraphBatchCall(newLinkRequest);
+    let newResponseBody = await handleErrorsJson(newResponse);
+
+    if (typeof newResponseBody === "string") {
+      newResponseBody = this.handleTextJsonResponseWithHtmlInBody(newResponseBody);
+    }
+
+    // Going recursive to fetch next link
+    const newSettledResponses = await this.fetchResponsesWithNextLink(newResponseBody.responses);
+    return [...alreadySuccess, ...newSettledResponses];
+  };
+
+  private fetchRequestWithRetryAfter = async (
+    originalRequests: IRequest[],
+    settledPromises: ISettledResponse[],
+    maxRetries: number,
+    retryCount = 0
+  ): Promise<IBatchResponse> => {
+    let retryAfterTimeout = 0;
+    if (retryCount >= maxRetries) {
+      return { responses: settledPromises };
+    }
+    const alreadySuccessRequest = [] as ISettledResponse[];
+    const failedRequest = [] as IRequest[];
+    settledPromises.forEach((item) => {
+      if (item.status === 200) {
+        alreadySuccessRequest.push(item);
+      } else if (item.status === 429) {
+        const newTimeout = Number(item.headers["Retry-After"]) * 1000 || 0;
+        retryAfterTimeout = newTimeout > retryAfterTimeout ? newTimeout : retryAfterTimeout;
+        failedRequest.push(originalRequests[Number(item.id)]);
+      }
+    });
+
+    if (failedRequest.length === 0) {
+      return { responses: alreadySuccessRequest };
+    }
+
+    // Await certain time from retry-after header
+    await new Promise((r) => setTimeout(r, retryAfterTimeout));
+
+    const newResponses = await this.apiGraphBatchCall(failedRequest);
+    let newResponseBody = await handleErrorsJson(newResponses);
+    if (typeof newResponseBody === "string") {
+      newResponseBody = this.handleTextJsonResponseWithHtmlInBody(newResponseBody);
+    }
+    const retryAfter = !!newResponseBody?.responses && this.findRetryAfterResponse(newResponseBody.responses);
+
+    if (retryAfter && newResponseBody.responses) {
+      newResponseBody = await this.fetchRequestWithRetryAfter(
+        failedRequest,
+        newResponseBody.responses,
+        maxRetries,
+        retryCount + 1
+      );
+    }
+    return { responses: [...alreadySuccessRequest, ...(newResponseBody?.responses || [])] };
+  };
+
+  private apiGraphBatchCall = async (requests: IRequest[]): Promise<Response> => {
+    try {
+      const response = await this.fetcher(`/$batch`, {
+        method: "POST",
+        body: JSON.stringify({ requests }),
+      });
+
+      return response;
+    } catch (error: any) {
+      throw new Error(error);
+    }
+  };
+
+  private handleTextJsonResponseWithHtmlInBody = (response: string): IBatchResponse => {
+    try {
+      const parsedJson = JSON.parse(response);
+      return parsedJson;
+    } catch (error) {
+      // Looking for html in body
+      const openTag = '"body":<';
+      const closeTag = "</html>";
+      const htmlBeginning = response.indexOf(openTag) + openTag.length - 1;
+      const htmlEnding = response.indexOf(closeTag) + closeTag.length + 2;
+      const resultString = `${response.repeat(1).substring(0, htmlBeginning)} ""${response
+        .repeat(1)
+        .substring(htmlEnding, response.length)}`;
+
+      return JSON.parse(resultString);
+    }
+  };
+
+  private findRetryAfterResponse = (response: ISettledResponse[]) => {
+    const foundRetry = response.find((request: ISettledResponse) => request.status === 429);
+    return !!foundRetry;
+  };
+
+  private processBusyTimes = (responses: ISettledResponse[]) => {
+    return responses.reduce(
+      (acc: BufferedBusyTime[], subResponse: { body: { value?: any[]; error?: any[] } }) => {
+        if (!subResponse.body?.value) return acc;
+        return acc.concat(
+          subResponse.body.value
+            .filter((evt) => evt.showAs !== "free" && evt.showAs !== "workingElsewhere")
+            .map((evt) => ({
+              start: evt.start.dateTime + "Z",
+              end: evt.end.dateTime + "Z",
+            }))
+        );
+      },
+      []
+    );
+  };
 }
@@ -0,0 +1,12 @@
+import { z } from "zod";
+
+import getParsedAppKeysFromSlug from "../../_utils/getParsedAppKeysFromSlug";
+
+const officeAppKeysSchema = z.object({
+  client_id: z.string(),
+  client_secret: z.string(),
+});
+
+export const getOfficeAppKeys = async () => {
+  return getParsedAppKeysFromSlug("office365-calendar", officeAppKeysSchema);
+};
@@ -100,11 +100,12 @@ const getCachedResults = async (
    const passedSelectedCalendars = selectedCalendars.filter((sc) => sc.integration === type);
    /** We extract external Ids so we don't cache too much */
    const selectedCalendarIds = passedSelectedCalendars.map((sc) => sc.externalId);
-    /** We create a unque hash key based on the input data */
+    /** We create a unique hash key based on the input data */
    const cacheKey = JSON.stringify({ id, selectedCalendarIds, dateFrom, dateTo });
    const cacheHashedKey = createHash("md5").update(cacheKey).digest("hex");
    /** Check if we already have cached data and return */
    const cachedAvailability = cache.get(cacheHashedKey);
+
    if (cachedAvailability) {
      log.debug(`Cache HIT: Calendar Availability for key: ${cacheKey}`);
      return cachedAvailability;
@@ -113,6 +114,7 @@ const getCachedResults = async (
    /** If we don't then we actually fetch external calendars (which can be very slow) */
    const availability = await c.getAvailability(dateFrom, dateTo, passedSelectedCalendars);
    /** We save the availability to a few seconds so recurrent calls are nearly instant */
+
    cache.put(cacheHashedKey, availability, CACHING_TIME);
    return availability;
  });
@@ -47,7 +47,7 @@ export async function getBusyTimes(params: {
  logger.debug(`prisma booking get took ${endPrismaBookingGet - startPrismaBookingGet}ms`);
  if (credentials.length > 0) {
    const calendarBusyTimes = await getBusyCalendarTimes(credentials, startTime, endTime, selectedCalendars);
-    // console.log("calendarBusyTimes", calendarBusyTimes);
+
    busyTimes.push(...calendarBusyTimes); /* 
    // TODO: Disabled until we can filter Zoom events by date. Also this is adding too much latency.
    const videoBusyTimes = (await getBusyVideoTimes(credentials)).filter(notEmpty);
@@ -151,6 +151,7 @@ export async function getUserAvailability(
  );
  const endGetWorkingHours = performance.now();
  logger.debug(`getWorkingHours took ${endGetWorkingHours - startGetWorkingHours}ms for userId ${userId}`);
+
  return {
    busy: bufferedBusyTimes,
    timeZone,
@@ -11,6 +11,9 @@ export function getErrorFromUnknown(cause: unknown): Error & { statusCode?: numb
 }

 export function handleErrorsJson(response: Response) {
+  if (response.headers.get("content-encoding") === "gzip") {
+    return response.text();
+  }
  if (response.status === 204) {
    return new Promise((resolve) => resolve({}));
  }
@@ -18,6 +21,7 @@ export function handleErrorsJson(response: Response) {
    response.json().then(console.log);
    throw Error(response.statusText);
  }
+
  return response.json();
 }

@@ -203,7 +203,7 @@ export async function getSchedule(
    })
  );

-  const workingHours = userSchedules.flatMap((s) => s.workingHours);
+  const workingHours = userSchedules?.flatMap((s) => s.workingHours);

  const slots: Record<string, Slot[]> = {};
  const availabilityCheckProps = {
@@ -279,6 +279,7 @@ export async function getSchedule(
    `checkForAvailability took ${checkForAvailabilityTime}ms and executed ${checkForAvailabilityCount} times`
  );
  logger.silly(`Available slots: ${JSON.stringify(slots)}`);
+
  return {
    slots,
  };