Skip to content

Commit

Permalink
chore: bump @clickhouse/client to v1.4.1 (#486)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wrn14897
wrn14897 authored Aug 8, 2024
1 parent 3b5ec64 commit f322f46
Show file tree
Hide file tree
Showing 6 changed files with 94 additions and 104 deletions.
5 changes: 5 additions & 0 deletions .changeset/tender-turkeys-pump.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@hyperdx/api': minor
---

chore: bump @clickhouse/client to v1.4.1
2 changes: 1 addition & 1 deletion packages/api/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"node": ">=18.12.0"
},
"dependencies": {
"@clickhouse/client": "^0.2.10",
"@clickhouse/client": "^1.4.1",
"@hyperdx/lucene": "^3.1.1",
"@hyperdx/node-opentelemetry": "^0.8.0",
"@opentelemetry/api": "^1.8.0",
Expand Down
169 changes: 77 additions & 92 deletions packages/api/src/clickhouse/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import {
BaseResultSet,
createClient,
ErrorLogParams as _CHErrorLogParams,
Logger as _CHLogger,
LogParams as _CHLogParams,
ResponseJSON,
ResultSet,
SettingsMap,
} from '@clickhouse/client';
import opentelemetry from '@opentelemetry/api';
Expand All @@ -13,7 +13,6 @@ import _ from 'lodash';
import ms from 'ms';
import { serializeError } from 'serialize-error';
import SqlString from 'sqlstring';
import { Readable } from 'stream';
import { z } from 'zod';

import * as config from '@/config';
Expand Down Expand Up @@ -158,10 +157,11 @@ export class CHLogger implements _CHLogger {

// TODO: move this to somewhere else
export const client = createClient({
host: config.CLICKHOUSE_HOST,
url: config.CLICKHOUSE_HOST,
username: config.CLICKHOUSE_USER,
password: config.CLICKHOUSE_PASSWORD,
request_timeout: ms('1m'),
application: 'hyperdx',
compression: {
request: false,
response: false, // has to be off to enable streaming
Expand All @@ -171,8 +171,7 @@ export const client = createClient({
// should be slightly less than the `keep_alive_timeout` setting in server's `config.xml`
// default is 3s there, so 2500 milliseconds seems to be a safe client value in this scenario
// another example: if your configuration has `keep_alive_timeout` set to 60s, you could put 59_000 here
socket_ttl: 60000,
retry_on_expired_socket: true,
idle_socket_ttl: 60000,
},
clickhouse_settings: {
connect_timeout: ms('1m') / 1000,
Expand Down Expand Up @@ -578,9 +577,7 @@ export const getCHServerMetrics = async () => {
query,
format: 'JSON',
});
const result = await rows.json<
ResponseJSON<{ metric: string; value: string }>
>();
const result = await rows.json<{ metric: string; value: string }>();
logger.info({
message: 'getCHServerMetrics',
query,
Expand Down Expand Up @@ -637,16 +634,14 @@ export const getMetricsTagsDEPRECATED = async ({
),
},
});
const result = await rows.json<
ResponseJSON<{
data_type: string;
is_delta: boolean;
is_monotonic: boolean;
name: string;
tags: Record<string, string>[];
unit: string;
}>
>();
const result = await rows.json<{
data_type: string;
is_delta: boolean;
is_monotonic: boolean;
name: string;
tags: Record<string, string>[];
unit: string;
}>();
logger.info({
message: 'getMetricsTagsDEPRECATED',
query,
Expand Down Expand Up @@ -711,15 +706,13 @@ export const getMetricsNames = async ({
),
},
});
const result = await rows.json<
ResponseJSON<{
data_type: string;
is_delta: boolean;
is_monotonic: boolean;
name: string;
unit: string;
}>
>();
const result = await rows.json<{
data_type: string;
is_delta: boolean;
is_monotonic: boolean;
name: string;
unit: string;
}>();
logger.info({
message: 'getMetricsNames',
query,
Expand Down Expand Up @@ -809,13 +802,11 @@ export const getMetricsTags = async ({
),
},
});
const result = await rows.json<
ResponseJSON<{
name: string;
data_type: string;
tags: Record<string, string>[];
}>
>();
const result = await rows.json<{
name: string;
data_type: string;
tags: Record<string, string>[];
}>();
logger.info({
message: 'getMetricsTags',
query,
Expand Down Expand Up @@ -1015,13 +1006,11 @@ export const getMetricsChart = async ({
),
},
});
const result = await rows.json<
ResponseJSON<{
data: number;
group: string;
ts_bucket: number;
}>
>();
const result = await rows.json<{
data: number;
group: string;
ts_bucket: number;
}>();
logger.info({
message: 'getMetricsChart',
query,
Expand Down Expand Up @@ -1745,13 +1734,11 @@ export const queryMultiSeriesChart = async ({
format: 'JSON',
});

const result = await rows.json<
ResponseJSON<{
ts_bucket: number;
group: string[];
[series_data: `series_${number}.data`]: number;
}>
>();
const result = await rows.json<{
ts_bucket: number;
group: string[];
[series_data: `series_${number}.data`]: number;
}>();
return result;
};

Expand Down Expand Up @@ -2081,13 +2068,11 @@ LIMIT ?`,
),
},
});
const result = await rows.json<
ResponseJSON<{
data: string;
ts_bucket: number;
group: string[];
}>
>();
const result = await rows.json<{
data: string;
ts_bucket: number;
group: string[];
}>();
return result;
} catch (e) {
span.recordException(e as any);
Expand Down Expand Up @@ -2253,15 +2238,13 @@ export const getLogsChart = async ({
),
},
});
const result = await rows.json<
ResponseJSON<{
data: string;
ts_bucket: number;
group: string;
rank: string;
rank_order_by_value: string;
}>
>();
const result = await rows.json<{
data: string;
ts_bucket: number;
group: string;
rank: string;
rank_order_by_value: string;
}>();
logger.info({
message: 'getChart',
query,
Expand Down Expand Up @@ -2347,7 +2330,7 @@ export const getChartHistogram = async ({
),
},
});
const result = await rows.json<ResponseJSON<Record<string, unknown>>>();
const result = await rows.json<Record<string, unknown>>();
logger.info({
message: 'getChartHistogram',
query,
Expand Down Expand Up @@ -2498,7 +2481,7 @@ WHERE sessions.sessionId IN (
),
},
});
const result = await rows.json<ResponseJSON<Record<string, unknown>>>();
const result = await rows.json<Record<string, unknown>>();
logger.info({
message: 'getSessions',
query: executedQuery,
Expand Down Expand Up @@ -2569,7 +2552,7 @@ export const getHistogram = async (
),
},
});
const result = await rows.json<ResponseJSON<Record<string, unknown>>>();
const result = await rows.json<Record<string, unknown>>();
logger.info({
message: 'getHistogram',
query,
Expand Down Expand Up @@ -2634,7 +2617,7 @@ export const getLogById = async (
),
},
});
const result = await rows.json<ResponseJSON<Record<string, unknown>>>();
const result = await rows.json<Record<string, unknown>>();
logger.info({
message: 'getLogById',
query,
Expand Down Expand Up @@ -2744,14 +2727,12 @@ export const checkAlert = async ({
),
},
});
const result = await rows.json<
ResponseJSON<{
data: string;
group?: string;
ts_bucket: number;
attributes: Record<string, string>;
}>
>();
const result = await rows.json<{
data: string;
group?: string;
ts_bucket: number;
attributes: Record<string, string>;
}>();
logger.info({
message: 'checkAlert',
query,
Expand Down Expand Up @@ -2924,7 +2905,7 @@ export const getLogBatchGroupedByBody = async ({
],
);

type Response = ResponseJSON<{
type Response = {
body: string;
buckets: string[];
ids: string[];
Expand All @@ -2933,9 +2914,9 @@ export const getLogBatchGroupedByBody = async ({
lines_count: string;
service: string;
timestamps: string[];
}>;
};

let result: Response;
let result: ResponseJSON<Response>;

await tracer.startActiveSpan(
'clickhouse.getLogBatchGroupedByBody',
Expand Down Expand Up @@ -3016,16 +2997,14 @@ export const getLogBatch = async ({
),
},
});
result = await rows.json<
ResponseJSON<{
id: string;
timestamp: string;
severity_text: string;
body: string;
_host: string;
_source: string;
}>
>();
result = await rows.json<{
id: string;
timestamp: string;
severity_text: string;
body: string;
_host: string;
_source: string;
}>();
span.setAttribute('results', result.data.length);
span.end();
});
Expand Down Expand Up @@ -3074,7 +3053,7 @@ export const getRrwebEvents = async ({
],
);

let resultSet: BaseResultSet<Readable>;
let resultSet: ResultSet<'JSONEachRow'>;
await tracer.startActiveSpan('clickhouse.getRrwebEvents', async span => {
span.setAttribute('query', query);

Expand All @@ -3090,7 +3069,12 @@ export const getRrwebEvents = async ({
});

// @ts-ignore
return resultSet.stream();
return resultSet.stream<{
b: string;
t: number;
ck: number;
tcks: number;
}>();
};

export const getLogStream = async ({
Expand Down Expand Up @@ -3133,7 +3117,7 @@ export const getLogStream = async ({
limit,
});

let resultSet: BaseResultSet<Readable>;
let resultSet: ResultSet<'JSONEachRow'>;
await tracer.startActiveSpan('clickhouse.getLogStream', async span => {
span.setAttribute('query', query);
span.setAttribute('search', q);
Expand All @@ -3160,6 +3144,7 @@ export const getLogStream = async ({
}
});

// TODO: type this ?
// @ts-ignore
return resultSet.stream();
};
2 changes: 1 addition & 1 deletion packages/api/src/routers/api/sessions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ router.get('/:sessionId/rrweb', async (req, res, next) => {
offset: offsetNum,
});

stream.on('data', (rows: Row[]) => {
stream.on('data', rows => {
res.write(`${rows.map(row => `data: ${row.text}`).join('\n')}\n\n`);
res.flush();
});
Expand Down
2 changes: 1 addition & 1 deletion packages/api/src/tasks/usageStats.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ const getClickhouseTableSize = async () => {
table3: clickhouse.TableName.Metric,
},
});
const result = await rows.json<ResponseJSON<any>>();
const result = await rows.json<any>();
return result.data;
};

Expand Down
Loading

0 comments on commit f322f46

Please sign in to comment.