diff options
Diffstat (limited to 'app')
28 files changed, 1128 insertions, 8 deletions
diff --git a/app/assets/javascripts/api.js b/app/assets/javascripts/api.js index dc6ea148047..c85e5b68f5f 100644 --- a/app/assets/javascripts/api.js +++ b/app/assets/javascripts/api.js @@ -492,6 +492,41 @@ const Api = { buildUrl(url) { return joinPaths(gon.relative_url_root || '', url.replace(':version', gon.api_version)); }, + + /** + * Returns pods logs for an environment with an optional pod and container + * + * @param {Object} params + * @param {Object} param.environment - Environment object + * @param {string=} params.podName - Pod name, if not set the backend assumes a default one + * @param {string=} params.containerName - Container name, if not set the backend assumes a default one + * @param {string=} params.start - Starting date to query the logs in ISO format + * @param {string=} params.end - Ending date to query the logs in ISO format + * @returns {Promise} Axios promise for the result of a GET request of logs + */ + getPodLogs({ environment, podName, containerName, search, start, end }) { + const url = this.buildUrl(environment.logs_api_path); + + const params = {}; + + if (podName) { + params.pod_name = podName; + } + if (containerName) { + params.container_name = containerName; + } + if (search) { + params.search = search; + } + if (start) { + params.start = start; + } + if (end) { + params.end = end; + } + + return axios.get(url, { params }); + }, }; export default Api; diff --git a/app/assets/javascripts/clusters_list/components/clusters.vue b/app/assets/javascripts/clusters_list/components/clusters.vue index f9f23fd556f..46dacf30f39 100644 --- a/app/assets/javascripts/clusters_list/components/clusters.vue +++ b/app/assets/javascripts/clusters_list/components/clusters.vue @@ -28,6 +28,10 @@ export default { label: __('Size'), }, { + key: 'cpu', + label: __('Total cores (vCPUs)'), + }, + { key: 'memory', label: __('Total memory (GB)'), }, diff --git a/app/assets/javascripts/create_cluster/components/cluster_form_dropdown.vue b/app/assets/javascripts/create_cluster/components/cluster_form_dropdown.vue index 2f7fcfcb755..e9d484bdd94 100644 --- a/app/assets/javascripts/create_cluster/components/cluster_form_dropdown.vue +++ b/app/assets/javascripts/create_cluster/components/cluster_form_dropdown.vue @@ -1,11 +1,12 @@ <script> +import { isNil } from 'lodash'; import $ from 'jquery'; import { GlIcon } from '@gitlab/ui'; import DropdownSearchInput from '~/vue_shared/components/dropdown/dropdown_search_input.vue'; import DropdownHiddenInput from '~/vue_shared/components/dropdown/dropdown_hidden_input.vue'; import DropdownButton from '~/vue_shared/components/dropdown/dropdown_button.vue'; -const toArray = value => [].concat(value); +const toArray = value => (isNil(value) ? [] : [].concat(value)); const itemsProp = (items, prop) => items.map(item => item[prop]); const defaultSearchFn = (searchQuery, labelProp) => item => item[labelProp].toLowerCase().indexOf(searchQuery) > -1; diff --git a/app/assets/javascripts/logs/components/environment_logs.vue b/app/assets/javascripts/logs/components/environment_logs.vue new file mode 100644 index 00000000000..b94cd2bcec4 --- /dev/null +++ b/app/assets/javascripts/logs/components/environment_logs.vue @@ -0,0 +1,222 @@ +<script> +import { mapActions, mapState, mapGetters } from 'vuex'; +import { GlDropdown, GlDropdownItem, GlFormGroup, GlSearchBoxByClick, GlAlert } from '@gitlab/ui'; +import DateTimePicker from '~/vue_shared/components/date_time_picker/date_time_picker.vue'; +import { scrollDown } from '~/lib/utils/scroll_utils'; +import LogControlButtons from './log_control_buttons.vue'; + +import { timeRanges, defaultTimeRange } from '~/monitoring/constants'; +import { timeRangeFromUrl } from '~/monitoring/utils'; + +export default { + components: { + GlAlert, + GlDropdown, + GlDropdownItem, + GlFormGroup, + GlSearchBoxByClick, + DateTimePicker, + LogControlButtons, + }, + props: { + environmentName: { + type: String, + required: false, + default: '', + }, + currentPodName: { + type: [String, null], + required: false, + default: null, + }, + environmentsPath: { + type: String, + required: false, + default: '', + }, + clusterApplicationsDocumentationPath: { + type: String, + required: true, + }, + }, + data() { + return { + searchQuery: '', + timeRanges, + isElasticStackCalloutDismissed: false, + }; + }, + computed: { + ...mapState('environmentLogs', ['environments', 'timeRange', 'logs', 'pods']), + ...mapGetters('environmentLogs', ['trace']), + + timeRangeModel: { + get() { + return this.timeRange.current; + }, + set(val) { + this.setTimeRange(val); + }, + }, + + showLoader() { + return this.logs.isLoading || !this.logs.isComplete; + }, + advancedFeaturesEnabled() { + const environment = this.environments.options.find( + ({ name }) => name === this.environments.current, + ); + return environment && environment.enable_advanced_logs_querying; + }, + disableAdvancedControls() { + return this.environments.isLoading || !this.advancedFeaturesEnabled; + }, + shouldShowElasticStackCallout() { + return !this.isElasticStackCalloutDismissed && this.disableAdvancedControls; + }, + }, + watch: { + trace(val) { + this.$nextTick(() => { + if (val) { + scrollDown(); + } + this.$refs.scrollButtons.update(); + }); + }, + }, + mounted() { + this.setInitData({ + timeRange: timeRangeFromUrl() || defaultTimeRange, + environmentName: this.environmentName, + podName: this.currentPodName, + }); + + this.fetchEnvironments(this.environmentsPath); + }, + methods: { + ...mapActions('environmentLogs', [ + 'setInitData', + 'setSearch', + 'setTimeRange', + 'showPodLogs', + 'showEnvironment', + 'fetchEnvironments', + ]), + }, +}; +</script> +<template> + <div class="build-page-pod-logs mt-3"> + <gl-alert + v-if="shouldShowElasticStackCallout" + class="mb-3 js-elasticsearch-alert" + @dismiss="isElasticStackCalloutDismissed = true" + > + {{ + s__( + 'Environments|Install Elastic Stack on your cluster to enable advanced querying capabilities such as full text search.', + ) + }} + <a :href="clusterApplicationsDocumentationPath"> + <strong> + {{ s__('View Documentation') }} + </strong> + </a> + </gl-alert> + <div class="top-bar js-top-bar d-flex"> + <div class="row mx-n1"> + <gl-form-group + id="environments-dropdown-fg" + :label="s__('Environments|Environment')" + label-size="sm" + label-for="environments-dropdown" + class="col-3 px-1" + > + <gl-dropdown + id="environments-dropdown" + :text="environments.current" + :disabled="environments.isLoading" + class="d-flex gl-h-32 js-environments-dropdown" + toggle-class="dropdown-menu-toggle" + > + <gl-dropdown-item + v-for="env in environments.options" + :key="env.id" + @click="showEnvironment(env.name)" + > + {{ env.name }} + </gl-dropdown-item> + </gl-dropdown> + </gl-form-group> + <gl-form-group + id="pods-dropdown-fg" + :label="s__('Environments|Logs from')" + label-size="sm" + label-for="pods-dropdown" + class="col-3 px-1" + > + <gl-dropdown + id="pods-dropdown" + :text="pods.current || s__('Environments|No pods to display')" + :disabled="environments.isLoading" + class="d-flex gl-h-32 js-pods-dropdown" + toggle-class="dropdown-menu-toggle" + > + <gl-dropdown-item + v-for="podName in pods.options" + :key="podName" + @click="showPodLogs(podName)" + > + {{ podName }} + </gl-dropdown-item> + </gl-dropdown> + </gl-form-group> + <gl-form-group + id="dates-fg" + :label="s__('Environments|Show last')" + label-size="sm" + label-for="time-window-dropdown" + class="col-3 px-1" + > + <date-time-picker + ref="dateTimePicker" + v-model="timeRangeModel" + class="w-100 gl-h-32" + :disabled="disableAdvancedControls" + :options="timeRanges" + /> + </gl-form-group> + <gl-form-group + id="search-fg" + :label="s__('Environments|Search')" + label-size="sm" + label-for="search" + class="col-3 px-1" + > + <gl-search-box-by-click + v-model.trim="searchQuery" + :disabled="disableAdvancedControls" + :placeholder="s__('Environments|Search')" + class="js-logs-search" + type="search" + autofocus + @submit="setSearch(searchQuery)" + /> + </gl-form-group> + </div> + + <log-control-buttons + ref="scrollButtons" + class="controllers align-self-end mb-1" + @refresh="showPodLogs(pods.current)" + /> + </div> + <pre class="build-trace js-log-trace"><code class="bash js-build-output">{{trace}} + <div v-if="showLoader" class="build-loader-animation js-build-loader-animation"> + <div class="dot"></div> + <div class="dot"></div> + <div class="dot"></div> + </div></code></pre> + </div> +</template> diff --git a/app/assets/javascripts/logs/components/log_control_buttons.vue b/app/assets/javascripts/logs/components/log_control_buttons.vue new file mode 100644 index 00000000000..d55c2f7cd4c --- /dev/null +++ b/app/assets/javascripts/logs/components/log_control_buttons.vue @@ -0,0 +1,93 @@ +<script> +import { GlButton, GlTooltipDirective } from '@gitlab/ui'; +import { + canScroll, + isScrolledToTop, + isScrolledToBottom, + scrollDown, + scrollUp, +} from '~/lib/utils/scroll_utils'; +import Icon from '~/vue_shared/components/icon.vue'; + +export default { + components: { + Icon, + GlButton, + }, + directives: { + GlTooltip: GlTooltipDirective, + }, + data() { + return { + scrollToTopEnabled: false, + scrollToBottomEnabled: false, + }; + }, + created() { + window.addEventListener('scroll', this.update); + }, + destroyed() { + window.removeEventListener('scroll', this.update); + }, + methods: { + /** + * Checks if page can be scrolled and updates + * enabled/disabled state of buttons accordingly + */ + update() { + this.scrollToTopEnabled = canScroll() && !isScrolledToTop(); + this.scrollToBottomEnabled = canScroll() && !isScrolledToBottom(); + }, + handleRefreshClick() { + this.$emit('refresh'); + }, + scrollUp, + scrollDown, + }, +}; +</script> + +<template> + <div> + <div + v-gl-tooltip + class="controllers-buttons" + :title="__('Scroll to top')" + aria-labelledby="scroll-to-top" + > + <gl-button + id="scroll-to-top" + class="btn-blank js-scroll-to-top" + :aria-label="__('Scroll to top')" + :disabled="!scrollToTopEnabled" + @click="scrollUp()" + ><icon name="scroll_up" + /></gl-button> + </div> + <div + v-gl-tooltip + class="controllers-buttons" + :title="__('Scroll to bottom')" + aria-labelledby="scroll-to-bottom" + > + <gl-button + id="scroll-to-bottom" + class="btn-blank js-scroll-to-bottom" + :aria-label="__('Scroll to bottom')" + :disabled="!scrollToBottomEnabled" + @click="scrollDown()" + ><icon name="scroll_down" + /></gl-button> + </div> + <gl-button + id="refresh-log" + v-gl-tooltip + class="ml-1 px-2 js-refresh-log" + :title="__('Refresh')" + :aria-label="__('Refresh')" + @click="handleRefreshClick" + > + <icon name="retry" /> + </gl-button> + </div> +</template> diff --git a/app/assets/javascripts/logs/index.js b/app/assets/javascripts/logs/index.js new file mode 100644 index 00000000000..70dbffdc3dd --- /dev/null +++ b/app/assets/javascripts/logs/index.js @@ -0,0 +1,24 @@ +import Vue from 'vue'; +import { getParameterValues } from '~/lib/utils/url_utility'; +import LogViewer from './components/environment_logs.vue'; +import store from './stores'; + +export default (props = {}) => { + const el = document.getElementById('environment-logs'); + const [currentPodName] = getParameterValues('pod_name'); + + // eslint-disable-next-line no-new + new Vue({ + el, + store, + render(createElement) { + return createElement(LogViewer, { + props: { + ...el.dataset, + currentPodName, + ...props, + }, + }); + }, + }); +}; diff --git a/app/assets/javascripts/logs/stores/actions.js b/app/assets/javascripts/logs/stores/actions.js new file mode 100644 index 00000000000..89a896b9dec --- /dev/null +++ b/app/assets/javascripts/logs/stores/actions.js @@ -0,0 +1,114 @@ +import Api from '~/api'; +import { backOff } from '~/lib/utils/common_utils'; +import httpStatusCodes from '~/lib/utils/http_status'; +import axios from '~/lib/utils/axios_utils'; +import flash from '~/flash'; +import { s__ } from '~/locale'; +import { convertToFixedRange } from '~/lib/utils/datetime_range'; + +import * as types from './mutation_types'; + +const flashTimeRangeWarning = () => { + flash(s__('Metrics|Invalid time range, please verify.'), 'warning'); +}; + +const flashLogsError = () => { + flash(s__('Metrics|There was an error fetching the logs, please try again')); +}; + +const requestLogsUntilData = params => + backOff((next, stop) => { + Api.getPodLogs(params) + .then(res => { + if (res.status === httpStatusCodes.ACCEPTED) { + next(); + return; + } + stop(res); + }) + .catch(err => { + stop(err); + }); + }); + +export const setInitData = ({ commit }, { timeRange, environmentName, podName }) => { + if (timeRange) { + commit(types.SET_TIME_RANGE, timeRange); + } + commit(types.SET_PROJECT_ENVIRONMENT, environmentName); + commit(types.SET_CURRENT_POD_NAME, podName); +}; + +export const showPodLogs = ({ dispatch, commit }, podName) => { + commit(types.SET_CURRENT_POD_NAME, podName); + dispatch('fetchLogs'); +}; + +export const setSearch = ({ dispatch, commit }, searchQuery) => { + commit(types.SET_SEARCH, searchQuery); + dispatch('fetchLogs'); +}; + +export const setTimeRange = ({ dispatch, commit }, timeRange) => { + commit(types.SET_TIME_RANGE, timeRange); + dispatch('fetchLogs'); +}; + +export const showEnvironment = ({ dispatch, commit }, environmentName) => { + commit(types.SET_PROJECT_ENVIRONMENT, environmentName); + commit(types.SET_CURRENT_POD_NAME, null); + dispatch('fetchLogs'); +}; + +export const fetchEnvironments = ({ commit, dispatch }, environmentsPath) => { + commit(types.REQUEST_ENVIRONMENTS_DATA); + + axios + .get(environmentsPath) + .then(({ data }) => { + commit(types.RECEIVE_ENVIRONMENTS_DATA_SUCCESS, data.environments); + dispatch('fetchLogs'); + }) + .catch(() => { + commit(types.RECEIVE_ENVIRONMENTS_DATA_ERROR); + flash(s__('Metrics|There was an error fetching the environments data, please try again')); + }); +}; + +export const fetchLogs = ({ commit, state }) => { + const params = { + environment: state.environments.options.find(({ name }) => name === state.environments.current), + podName: state.pods.current, + search: state.search, + }; + + if (state.timeRange.current) { + try { + const { start, end } = convertToFixedRange(state.timeRange.current); + params.start = start; + params.end = end; + } catch { + flashTimeRangeWarning(); + } + } + + commit(types.REQUEST_PODS_DATA); + commit(types.REQUEST_LOGS_DATA); + + return requestLogsUntilData(params) + .then(({ data }) => { + const { pod_name, pods, logs } = data; + commit(types.SET_CURRENT_POD_NAME, pod_name); + + commit(types.RECEIVE_PODS_DATA_SUCCESS, pods); + commit(types.RECEIVE_LOGS_DATA_SUCCESS, logs); + }) + .catch(() => { + commit(types.RECEIVE_PODS_DATA_ERROR); + commit(types.RECEIVE_LOGS_DATA_ERROR); + flashLogsError(); + }); +}; + +// prevent babel-plugin-rewire from generating an invalid default during karma tests +export default () => {}; diff --git a/app/assets/javascripts/logs/stores/getters.js b/app/assets/javascripts/logs/stores/getters.js new file mode 100644 index 00000000000..c7dbb72ce3d --- /dev/null +++ b/app/assets/javascripts/logs/stores/getters.js @@ -0,0 +1,9 @@ +import dateFormat from 'dateformat'; + +export const trace = state => + state.logs.lines + .map(item => [dateFormat(item.timestamp, 'UTC:mmm dd HH:MM:ss.l"Z"'), item.message].join(' | ')) + .join('\n'); + +// prevent babel-plugin-rewire from generating an invalid default during karma tests +export default () => {}; diff --git a/app/assets/javascripts/logs/stores/index.js b/app/assets/javascripts/logs/stores/index.js new file mode 100644 index 00000000000..d16941ddf93 --- /dev/null +++ b/app/assets/javascripts/logs/stores/index.js @@ -0,0 +1,23 @@ +import Vue from 'vue'; +import Vuex from 'vuex'; +import * as actions from './actions'; +import * as getters from './getters'; +import mutations from './mutations'; +import state from './state'; + +Vue.use(Vuex); + +export const createStore = () => + new Vuex.Store({ + modules: { + environmentLogs: { + namespaced: true, + actions, + mutations, + state: state(), + getters, + }, + }, + }); + +export default createStore; diff --git a/app/assets/javascripts/logs/stores/mutation_types.js b/app/assets/javascripts/logs/stores/mutation_types.js new file mode 100644 index 00000000000..b8e70f95d92 --- /dev/null +++ b/app/assets/javascripts/logs/stores/mutation_types.js @@ -0,0 +1,16 @@ +export const SET_PROJECT_ENVIRONMENT = 'SET_PROJECT_ENVIRONMENT'; +export const SET_SEARCH = 'SET_SEARCH'; +export const SET_TIME_RANGE = 'SET_TIME_RANGE'; +export const SET_CURRENT_POD_NAME = 'SET_CURRENT_POD_NAME'; + +export const REQUEST_ENVIRONMENTS_DATA = 'REQUEST_ENVIRONMENTS_DATA'; +export const RECEIVE_ENVIRONMENTS_DATA_SUCCESS = 'RECEIVE_ENVIRONMENTS_DATA_SUCCESS'; +export const RECEIVE_ENVIRONMENTS_DATA_ERROR = 'RECEIVE_ENVIRONMENTS_DATA_ERROR'; + +export const REQUEST_LOGS_DATA = 'REQUEST_LOGS_DATA'; +export const RECEIVE_LOGS_DATA_SUCCESS = 'RECEIVE_LOGS_DATA_SUCCESS'; +export const RECEIVE_LOGS_DATA_ERROR = 'RECEIVE_LOGS_DATA_ERROR'; + +export const REQUEST_PODS_DATA = 'REQUEST_PODS_DATA'; +export const RECEIVE_PODS_DATA_SUCCESS = 'RECEIVE_PODS_DATA_SUCCESS'; +export const RECEIVE_PODS_DATA_ERROR = 'RECEIVE_PODS_DATA_ERROR'; diff --git a/app/assets/javascripts/logs/stores/mutations.js b/app/assets/javascripts/logs/stores/mutations.js new file mode 100644 index 00000000000..ca31dd3bc20 --- /dev/null +++ b/app/assets/javascripts/logs/stores/mutations.js @@ -0,0 +1,61 @@ +import * as types from './mutation_types'; + +export default { + /** Search data */ + [types.SET_SEARCH](state, searchQuery) { + state.search = searchQuery; + }, + + /** Time Range data */ + [types.SET_TIME_RANGE](state, timeRange) { + state.timeRange.current = timeRange; + }, + + /** Environments data */ + [types.SET_PROJECT_ENVIRONMENT](state, environmentName) { + state.environments.current = environmentName; + }, + [types.REQUEST_ENVIRONMENTS_DATA](state) { + state.environments.options = []; + state.environments.isLoading = true; + }, + [types.RECEIVE_ENVIRONMENTS_DATA_SUCCESS](state, environmentOptions) { + state.environments.options = environmentOptions; + state.environments.isLoading = false; + }, + [types.RECEIVE_ENVIRONMENTS_DATA_ERROR](state) { + state.environments.options = []; + state.environments.isLoading = false; + }, + + /** Logs data */ + [types.REQUEST_LOGS_DATA](state) { + state.logs.lines = []; + state.logs.isLoading = true; + state.logs.isComplete = false; + }, + [types.RECEIVE_LOGS_DATA_SUCCESS](state, lines) { + state.logs.lines = lines; + state.logs.isLoading = false; + state.logs.isComplete = true; + }, + [types.RECEIVE_LOGS_DATA_ERROR](state) { + state.logs.lines = []; + state.logs.isLoading = false; + state.logs.isComplete = true; + }, + + /** Pods data */ + [types.SET_CURRENT_POD_NAME](state, podName) { + state.pods.current = podName; + }, + [types.REQUEST_PODS_DATA](state) { + state.pods.options = []; + }, + [types.RECEIVE_PODS_DATA_SUCCESS](state, podOptions) { + state.pods.options = podOptions; + }, + [types.RECEIVE_PODS_DATA_ERROR](state) { + state.pods.options = []; + }, +}; diff --git a/app/assets/javascripts/logs/stores/state.js b/app/assets/javascripts/logs/stores/state.js new file mode 100644 index 00000000000..eaf1b1bdd93 --- /dev/null +++ b/app/assets/javascripts/logs/stores/state.js @@ -0,0 +1,42 @@ +import { timeRanges, defaultTimeRange } from '~/monitoring/constants'; + +export default () => ({ + /** + * Full text search + */ + search: '', + + /** + * Time range (Show last) + */ + timeRange: { + options: timeRanges, + current: defaultTimeRange, + }, + + /** + * Environments list information + */ + environments: { + options: [], + isLoading: false, + current: null, + }, + + /** + * Logs including trace + */ + logs: { + lines: [], + isLoading: false, + isComplete: true, + }, + + /** + * Pods list information + */ + pods: { + options: [], + current: null, + }, +}); diff --git a/app/assets/javascripts/logs/utils.js b/app/assets/javascripts/logs/utils.js new file mode 100644 index 00000000000..668efee74e8 --- /dev/null +++ b/app/assets/javascripts/logs/utils.js @@ -0,0 +1,23 @@ +import { secondsToMilliseconds } from '~/lib/utils/datetime_utility'; + +/** + * Returns a time range (`start`, `end`) where `start` is the + * current time minus a given number of seconds and `end` + * is the current time (`now()`). + * + * @param {Number} seconds Seconds duration, defaults to 0. + * @returns {Object} range Time range + * @returns {String} range.start ISO String of current time minus given seconds + * @returns {String} range.end ISO String of current time + */ +export const getTimeRange = (seconds = 0) => { + const end = Math.floor(Date.now() / 1000); // convert milliseconds to seconds + const start = end - seconds; + + return { + start: new Date(secondsToMilliseconds(start)).toISOString(), + end: new Date(secondsToMilliseconds(end)).toISOString(), + }; +}; + +export default {}; diff --git a/app/assets/javascripts/pages/projects/logs/index.js b/app/assets/javascripts/pages/projects/logs/index.js new file mode 100644 index 00000000000..36747069ebb --- /dev/null +++ b/app/assets/javascripts/pages/projects/logs/index.js @@ -0,0 +1,3 @@ +import logsBundle from '~/logs'; + +document.addEventListener('DOMContentLoaded', logsBundle); diff --git a/app/assets/stylesheets/framework/common.scss b/app/assets/stylesheets/framework/common.scss index 20846502e85..4d8ae8a5652 100644 --- a/app/assets/stylesheets/framework/common.scss +++ b/app/assets/stylesheets/framework/common.scss @@ -524,6 +524,8 @@ img.emoji { cursor: pointer; } +.cursor-not-allowed { cursor: not-allowed; } + // this needs to use "!important" due to some very specific styles // around buttons .cursor-default { diff --git a/app/assets/stylesheets/pages/builds.scss b/app/assets/stylesheets/pages/builds.scss index 0db90fc88fc..59266af96b4 100644 --- a/app/assets/stylesheets/pages/builds.scss +++ b/app/assets/stylesheets/pages/builds.scss @@ -357,3 +357,42 @@ } } } + +.build-page-pod-logs { + .build-trace-container { + position: relative; + } + + .build-trace { + @include build-trace(); + } + + .top-bar { + @include build-trace-top-bar($gl-line-height * 5); + + .dropdown-menu-toggle { + width: 200px; + + @include media-breakpoint-up(sm) { + width: 300px; + } + } + + .controllers { + @include build-controllers(16px, flex-end, true, 2); + } + + .refresh-control { + @include build-controllers(16px, flex-end, true, 0); + margin-left: 2px; + } + } + + .btn-refresh svg { + top: 0; + } + + .build-loader-animation { + @include build-loader-animation; + } +} diff --git a/app/controllers/projects/logs_controller.rb b/app/controllers/projects/logs_controller.rb new file mode 100644 index 00000000000..1b0fdf2a337 --- /dev/null +++ b/app/controllers/projects/logs_controller.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Projects + class LogsController < Projects::ApplicationController + before_action :authorize_read_pod_logs! + before_action :environment + before_action :ensure_deployments, only: %i(k8s elasticsearch) + + def index + if environment.nil? + render :empty_logs + else + render :index + end + end + + def k8s + render_logs(::PodLogs::KubernetesService, k8s_params) + end + + def elasticsearch + render_logs(::PodLogs::ElasticsearchService, elasticsearch_params) + end + + private + + def render_logs(service, permitted_params) + ::Gitlab::UsageCounters::PodLogs.increment(project.id) + ::Gitlab::PollingInterval.set_header(response, interval: 3_000) + + result = service.new(cluster, namespace, params: permitted_params).execute + + if result.nil? + head :accepted + elsif result[:status] == :success + render json: result + else + render status: :bad_request, json: result + end + end + + def index_params + params.permit(:environment_name) + end + + def k8s_params + params.permit(:container_name, :pod_name) + end + + def elasticsearch_params + params.permit(:container_name, :pod_name, :search, :start, :end) + end + + def environment + @environment ||= if index_params.key?(:environment_name) + EnvironmentsFinder.new(project, current_user, name: index_params[:environment_name]).find.first + else + project.default_environment + end + end + + def cluster + environment.deployment_platform&.cluster + end + + def namespace + environment.deployment_namespace + end + + def ensure_deployments + return if cluster && namespace.present? + + render status: :bad_request, json: { + status: :error, + message: _('Environment does not have deployments') + } + end + end +end diff --git a/app/helpers/environments_helper.rb b/app/helpers/environments_helper.rb index fd330d4efd9..6bf920448a5 100644 --- a/app/helpers/environments_helper.rb +++ b/app/helpers/environments_helper.rb @@ -41,4 +41,13 @@ module EnvironmentsHelper "external-dashboard-url" => project.metrics_setting_external_dashboard_url } end + + def environment_logs_data(project, environment) + { + "environment-name": environment.name, + "environments-path": project_environments_path(project, format: :json), + "environment-id": environment.id, + "cluster-applications-documentation-path" => help_page_path('user/clusters/applications.md', anchor: 'elastic-stack') + } + end end diff --git a/app/models/environment.rb b/app/models/environment.rb index 0e2962b893a..3f9247b1544 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -330,6 +330,10 @@ class Environment < ApplicationRecord self.auto_stop_at = parsed_result.seconds.from_now end + def elastic_stack_available? + !!deployment_platform&.cluster&.application_elastic_stack&.available? + end + private def has_metrics_and_can_query? diff --git a/app/models/snippet_repository.rb b/app/models/snippet_repository.rb index 70f26001b5f..f879f58b5a3 100644 --- a/app/models/snippet_repository.rb +++ b/app/models/snippet_repository.rb @@ -18,12 +18,6 @@ class SnippetRepository < ApplicationRecord end end - def create_file(user, path, content, **options) - options[:actions] = transform_file_entries([{ file_path: path, content: content }]) - - capture_git_error { repository.multi_action(user, **options) } - end - def multi_files_action(user, files = [], **options) return if files.nil? || files.empty? diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 15d60fe9cd8..95b92d4c108 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -314,6 +314,7 @@ class ProjectPolicy < BasePolicy enable :admin_operations enable :read_deploy_token enable :create_deploy_token + enable :read_pod_logs end rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror diff --git a/app/serializers/environment_entity.rb b/app/serializers/environment_entity.rb index 74d6806e83f..d9af7af8a8b 100644 --- a/app/serializers/environment_entity.rb +++ b/app/serializers/environment_entity.rb @@ -47,6 +47,22 @@ class EnvironmentEntity < Grape::Entity environment.available? && can?(current_user, :stop_environment, environment) end + expose :logs_path, if: -> (*) { can_read_pod_logs? } do |environment| + project_logs_path(environment.project, environment_name: environment.name) + end + + expose :logs_api_path, if: -> (*) { can_read_pod_logs? } do |environment| + if environment.elastic_stack_available? + elasticsearch_project_logs_path(environment.project, environment_name: environment.name, format: :json) + else + k8s_project_logs_path(environment.project, environment_name: environment.name, format: :json) + end + end + + expose :enable_advanced_logs_querying, if: -> (*) { can_read_pod_logs? } do |environment| + environment.elastic_stack_available? + end + private alias_method :environment, :object @@ -63,6 +79,10 @@ class EnvironmentEntity < Grape::Entity can?(current_user, :update_environment, environment) end + def can_read_pod_logs? + can?(current_user, :read_pod_logs, environment.project) + end + def cluster_platform_kubernetes? deployment_platform && deployment_platform.is_a?(Clusters::Platforms::Kubernetes) end diff --git a/app/services/pod_logs/base_service.rb b/app/services/pod_logs/base_service.rb new file mode 100644 index 00000000000..668ee6b88a8 --- /dev/null +++ b/app/services/pod_logs/base_service.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module PodLogs + class BaseService < ::BaseService + include ReactiveCaching + include Stepable + + attr_reader :cluster, :namespace, :params + + CACHE_KEY_GET_POD_LOG = 'get_pod_log' + K8S_NAME_MAX_LENGTH = 253 + + SUCCESS_RETURN_KEYS = %i(status logs pod_name container_name pods).freeze + + def id + cluster.id + end + + def initialize(cluster, namespace, params: {}) + @cluster = cluster + @namespace = namespace + @params = filter_params(params.dup.stringify_keys).to_hash + end + + def execute + with_reactive_cache( + CACHE_KEY_GET_POD_LOG, + namespace, + params + ) do |result| + result + end + end + + def calculate_reactive_cache(request, _namespace, _params) + case request + when CACHE_KEY_GET_POD_LOG + execute_steps + else + exception = StandardError.new('Unknown reactive cache request') + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception, request: request) + error(_('Unknown cache key')) + end + end + + private + + def valid_params + %w(pod_name container_name) + end + + def check_arguments(result) + return error(_('Cluster does not exist')) if cluster.nil? + return error(_('Namespace is empty')) if namespace.blank? + + success(result) + end + + def check_param_lengths(_result) + pod_name = params['pod_name'].presence + container_name = params['container_name'].presence + + if pod_name&.length.to_i > K8S_NAME_MAX_LENGTH + return error(_('pod_name cannot be larger than %{max_length}'\ + ' chars' % { max_length: K8S_NAME_MAX_LENGTH })) + elsif container_name&.length.to_i > K8S_NAME_MAX_LENGTH + return error(_('container_name cannot be larger than'\ + ' %{max_length} chars' % { max_length: K8S_NAME_MAX_LENGTH })) + end + + success(pod_name: pod_name, container_name: container_name) + end + + def get_raw_pods(result) + result[:raw_pods] = cluster.kubeclient.get_pods(namespace: namespace) + + success(result) + end + + def get_pod_names(result) + result[:pods] = result[:raw_pods].map(&:metadata).map(&:name) + + success(result) + end + + def check_pod_name(result) + # If pod_name is not received as parameter, get the pod logs of the first + # pod of this namespace. + result[:pod_name] ||= result[:pods].first + + unless result[:pod_name] + return error(_('No pods available')) + end + + unless result[:pods].include?(result[:pod_name]) + return error(_('Pod does not exist')) + end + + success(result) + end + + def check_container_name(result) + pod_details = result[:raw_pods].first { |p| p.metadata.name == result[:pod_name] } + containers = pod_details.spec.containers.map(&:name) + + # select first container if not specified + result[:container_name] ||= containers.first + + unless result[:container_name] + return error(_('No containers available')) + end + + unless containers.include?(result[:container_name]) + return error(_('Container does not exist')) + end + + success(result) + end + + def pod_logs(result) + raise NotImplementedError + end + + def filter_return_keys(result) + result.slice(*SUCCESS_RETURN_KEYS) + end + + def filter_params(params) + params.slice(*valid_params) + end + end +end diff --git a/app/services/pod_logs/elasticsearch_service.rb b/app/services/pod_logs/elasticsearch_service.rb new file mode 100644 index 00000000000..7524bf7ce10 --- /dev/null +++ b/app/services/pod_logs/elasticsearch_service.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module PodLogs + class ElasticsearchService < BaseService + steps :check_arguments, + :check_param_lengths, + :get_raw_pods, + :get_pod_names, + :check_pod_name, + :check_container_name, + :check_times, + :check_search, + :pod_logs, + :filter_return_keys + + self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) } + + private + + def valid_params + %w(pod_name container_name search start end) + end + + def check_times(result) + result[:start] = params['start'] if params.key?('start') && Time.iso8601(params['start']) + result[:end] = params['end'] if params.key?('end') && Time.iso8601(params['end']) + + success(result) + rescue ArgumentError + error(_('Invalid start or end time format')) + end + + def check_search(result) + result[:search] = params['search'] if params.key?('search') + + success(result) + end + + def pod_logs(result) + client = cluster&.application_elastic_stack&.elasticsearch_client + return error(_('Unable to connect to Elasticsearch')) unless client + + result[:logs] = ::Gitlab::Elasticsearch::Logs.new(client).pod_logs( + namespace, + result[:pod_name], + result[:container_name], + result[:search], + result[:start], + result[:end] + ) + + success(result) + rescue Elasticsearch::Transport::Transport::ServerError => e + ::Gitlab::ErrorTracking.track_exception(e) + + error(_('Elasticsearch returned status code: %{status_code}') % { + # ServerError is the parent class of exceptions named after HTTP status codes, eg: "Elasticsearch::Transport::Transport::Errors::NotFound" + # there is no method on the exception other than the class name to determine the type of error encountered. + status_code: e.class.name.split('::').last + }) + end + end +end diff --git a/app/services/pod_logs/kubernetes_service.rb b/app/services/pod_logs/kubernetes_service.rb new file mode 100644 index 00000000000..8f12b364e73 --- /dev/null +++ b/app/services/pod_logs/kubernetes_service.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module PodLogs + class KubernetesService < BaseService + LOGS_LIMIT = 500.freeze + REPLACEMENT_CHAR = "\u{FFFD}" + + EncodingHelperError = Class.new(StandardError) + + steps :check_arguments, + :check_param_lengths, + :get_raw_pods, + :get_pod_names, + :check_pod_name, + :check_container_name, + :pod_logs, + :encode_logs_to_utf8, + :split_logs, + :filter_return_keys + + self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) } + + private + + def pod_logs(result) + result[:logs] = cluster.kubeclient.get_pod_log( + result[:pod_name], + namespace, + container: result[:container_name], + tail_lines: LOGS_LIMIT, + timestamps: true + ).body + + success(result) + rescue Kubeclient::ResourceNotFoundError + error(_('Pod not found')) + rescue Kubeclient::HttpError => e + ::Gitlab::ErrorTracking.track_exception(e) + + error(_('Kubernetes API returned status code: %{error_code}') % { + error_code: e.error_code + }) + end + + # Check https://gitlab.com/gitlab-org/gitlab/issues/34965#note_292261879 + # for more details on why this is necessary. + def encode_logs_to_utf8(result) + return success(result) if result[:logs].nil? + return success(result) if result[:logs].encoding == Encoding::UTF_8 + + result[:logs] = encode_utf8(result[:logs]) + + success(result) + rescue EncodingHelperError + error(_('Unable to convert Kubernetes logs encoding to UTF-8')) + end + + def split_logs(result) + result[:logs] = result[:logs].strip.lines(chomp: true).map do |line| + # message contains a RFC3339Nano timestamp, then a space, then the log line. + # resolution of the nanoseconds can vary, so we split on the first space + values = line.split(' ', 2) + { + timestamp: values[0], + message: values[1] + } + end + + success(result) + end + + def encode_utf8(logs) + utf8_logs = Gitlab::EncodingHelper.encode_utf8(logs.dup, replace: REPLACEMENT_CHAR) + + # Gitlab::EncodingHelper.encode_utf8 can return '' or nil if an exception + # is raised while encoding. We prefer to return an error rather than wrongly + # display blank logs. + no_utf8_logs = logs.present? && utf8_logs.blank? + unexpected_encoding = utf8_logs&.encoding != Encoding::UTF_8 + + if no_utf8_logs || unexpected_encoding + raise EncodingHelperError, 'Could not convert Kubernetes logs to UTF-8' + end + + utf8_logs + end + end +end diff --git a/app/views/layouts/nav/sidebar/_project.html.haml b/app/views/layouts/nav/sidebar/_project.html.haml index 5afe43d6636..aef9532fd46 100644 --- a/app/views/layouts/nav/sidebar/_project.html.haml +++ b/app/views/layouts/nav/sidebar/_project.html.haml @@ -263,7 +263,11 @@ %span = _('Serverless') - = render_if_exists 'layouts/nav/sidebar/pod_logs_link' # EE-specific + - if project_nav_tab?(:environments) && can?(current_user, :read_pod_logs, @project) + = nav_link(controller: :logs, action: [:index]) do + = link_to project_logs_path(@project), title: _('Logs') do + %span + = _('Logs') - if project_nav_tab? :clusters - show_cluster_hint = show_gke_cluster_integration_callout?(@project) diff --git a/app/views/projects/logs/empty_logs.html.haml b/app/views/projects/logs/empty_logs.html.haml new file mode 100644 index 00000000000..52598e0be8d --- /dev/null +++ b/app/views/projects/logs/empty_logs.html.haml @@ -0,0 +1,14 @@ +- page_title _('Logs') + +.row.empty-state + .col-sm-12 + .svg-content + = image_tag 'illustrations/operations_log_pods_empty.svg' + .col-12 + .text-content + %h4.text-center + = s_('Environments|No deployed environments') + %p.state-description.text-center + = s_('Logs|To see the logs, deploy your code to an environment.') + .text-center + = link_to s_('Environments|Learn about environments'), help_page_path('ci/environments'), class: 'btn btn-success' diff --git a/app/views/projects/logs/index.html.haml b/app/views/projects/logs/index.html.haml new file mode 100644 index 00000000000..1f74eb52fd9 --- /dev/null +++ b/app/views/projects/logs/index.html.haml @@ -0,0 +1 @@ +#environment-logs{ data: environment_logs_data(@project, @environment) } |