diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-28 06:08:41 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-28 06:08:41 +0300 |
commit | a22f031743db01e51ee4f85c3368aa21cf8ac729 (patch) | |
tree | 58b4aabab8e8acfd684b63b05da2cd2cb6a177a6 | |
parent | d690a8d62bc0abd5ab937822ff7a1b9f1e7b1a91 (diff) |
Add latest changes from gitlab-org/gitlab@master
21 files changed, 522 insertions, 388 deletions
diff --git a/GITLAB_KAS_VERSION b/GITLAB_KAS_VERSION index 1750564f270..d169b2f2d0a 100644 --- a/GITLAB_KAS_VERSION +++ b/GITLAB_KAS_VERSION @@ -1 +1 @@ -0.0.6 +0.0.8 diff --git a/app/services/import/github_service.rb b/app/services/import/github_service.rb index a2923b1e4f9..948dba2d206 100644 --- a/app/services/import/github_service.rb +++ b/app/services/import/github_service.rb @@ -6,6 +6,10 @@ module Import attr_reader :params, :current_user def execute(access_params, provider) + if blocked_url? + return log_and_return_error("Invalid URL: #{url}", :bad_request) + end + unless authorized? return error(_('This namespace has already been taken! Please choose another one.'), :unprocessable_entity) end @@ -56,6 +60,25 @@ module Import can?(current_user, :create_projects, target_namespace) end + def url + @url ||= params[:github_hostname] + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def blocked_url? + Gitlab::UrlBlocker.blocked_url?( + url, + { + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests?, + schemes: %w(http https) + } + ) + end + private def log_error(exception) diff --git a/changelogs/unreleased/add-github-host-to-api.yml b/changelogs/unreleased/add-github-host-to-api.yml new file mode 100644 index 00000000000..9c31457db04 --- /dev/null +++ b/changelogs/unreleased/add-github-host-to-api.yml @@ -0,0 +1,5 @@ +--- +title: Add hostname to GitHub import API +merge_request: 45188 +author: +type: added diff --git a/doc/api/README.md b/doc/api/README.md index 3f7dae055e2..c15687ce6ed 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -167,7 +167,7 @@ to authenticate with the API: - [Conan Repository](../user/packages/conan_repository/index.md) - [Container Registry](../user/packages/container_registry/index.md) (`$CI_REGISTRY_PASSWORD` is actually `$CI_JOB_TOKEN`, but this may change in the future) - [Go Proxy](../user/packages/go_proxy/index.md) - - [Maven Repository](../user/packages/maven_repository/index.md#authenticate-with-a-ci-job-token) + - [Maven Repository](../user/packages/maven_repository/index.md#authenticate-with-a-ci-job-token-in-maven) - [NPM Repository](../user/packages/npm_registry/index.md#authenticating-with-a-ci-job-token) - [Nuget Repository](../user/packages/nuget_repository/index.md) - [PyPI Repository](../user/packages/pypi_repository/index.md#using-gitlab-ci-with-pypi-packages) diff --git a/doc/api/import.md b/doc/api/import.md index 17abb90d8e7..691e042084e 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -14,6 +14,7 @@ POST /import/github | `repo_id` | integer | yes | GitHub repository ID | | `new_name` | string | no | New repository name | | `target_namespace` | string | yes | Namespace to import repository into. Supports subgroups like `/namespace/subgroup`. | +| `github_hostname` | string | no | Custom GitHub enterprise hostname. Defaults to GitHub.com if `github_hostname` is not set. | ```shell curl --request POST \ diff --git a/doc/ci/pipelines/pipeline_efficiency.md b/doc/ci/pipelines/pipeline_efficiency.md index c4febba8f44..149c596430c 100644 --- a/doc/ci/pipelines/pipeline_efficiency.md +++ b/doc/ci/pipelines/pipeline_efficiency.md @@ -222,6 +222,8 @@ Methods to reduce Docker image size: - Create a dedicated development image. - Disable man pages and docs installed by packages to save space. - Reduce the `RUN` layers and combine software installation steps. +- Use [multi-stage builds](https://blog.alexellis.io/mutli-stage-docker-builds/) + to merge multiple Dockerfiles that use the builder pattern into one Dockerfile, which can reduce image size. - If using `apt`, add `--no-install-recommends` to avoid unnecessary packages. - Clean up caches and files that are no longer needed at the end. For example `rm -rf /var/lib/apt/lists/*` for Debian and Ubuntu, or `yum clean all` for RHEL and CentOS. diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 51053e37dae..90ccbe68847 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -1746,6 +1746,21 @@ The voting strategy (in GitLab 13.4 and later) requires the primary and secondary voters to agree. ``` +#### End-of-life for features or products + +Whenever a feature or product enters the end-of-life process, indicate its +status by using the `Danger` [alert](#alert-boxes) with the `**Important**` +keyword directly below the feature or product's header (which can include H1 +page titles). Link to the deprecation and removal issues, if possible. + +For example: + +```markdown +DANGER: **Important:** +This feature is in its end-of-life process. It is [deprecated](link-to-issue) +for use in GitLab X.X, and is planned for [removal](link-to-issue) in GitLab X.X. +``` + ### Versions in the past or future When describing functionality available in past or future versions, use: diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index d4a8ff0cdb4..5f4078c9d71 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -17,9 +17,9 @@ Then, install the packages whenever you need to use them as a dependency. This section explains how to install Maven and build a package. If you already use Maven and know how to build your own packages, go to the -[next section](#add-the-package-registry-as-a-maven-remote). +[next section](#authenticate-to-the-package-registry-with-maven). -Maven repositories work well with Gradle, too. To set up a Gradle project, see [get started with Gradle](#use-gradle-to-create-a-java-project). +Maven repositories work well with Gradle, too. To set up a Gradle project, see [get started with Gradle](#build-a-java-project-with-gradle). ### Install Maven @@ -29,14 +29,14 @@ The required minimum versions are: - Maven 3.6+ Follow the instructions at [maven.apache.org](https://maven.apache.org/install.html) -to download and install Maven for your local development environment. Once +to download and install Maven for your local development environment. After installation is complete, verify you can use Maven in your terminal by running: ```shell mvn --version ``` -You should see something similar to the below printed in the output: +The output should be similar to: ```shell Apache Maven 3.6.1 (d66c9c0b3152b2e69ee9bac180bb8fcc8e6af555; 2019-04-04T20:00:29+01:00) @@ -48,29 +48,26 @@ OS name: "mac os x", version: "10.15.2", arch: "x86_64", family: "mac" ### Create a project -Understanding how to create a full Java project is outside the scope of this -guide but you can follow the steps below to create a new project that can be +Follow these steps to create a Maven project that can be published to the GitLab Package Registry. -Start by opening your terminal and creating a directory where you would like to -store the project in your environment. From inside the directory, you can run -the following Maven command to initialize a new package: +1. Open your terminal and create a directory to store the project. +1. From the new directory, run this Maven command to initialize a new package: -```shell -mvn archetype:generate -DgroupId=com.mycompany.mydepartment -DartifactId=my-project -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false -``` + ```shell + mvn archetype:generate -DgroupId=com.mycompany.mydepartment -DartifactId=my-project -DarchetypeArtifactId=maven-archetype-quickstart -DinteractiveMode=false + ``` -The arguments are as follows: + The arguments are: -- `DgroupId`: A unique string that identifies your package. You should follow -the [Maven naming conventions](https://maven.apache.org/guides/mini/guide-naming-conventions.html). -- `DartifactId`: The name of the JAR, appended to the end of the `DgroupId`. -- `DarchetypeArtifactId`: The archetype used to create the initial structure of -the project. -- `DinteractiveMode`: Create the project using batch mode (optional). + - `DgroupId`: A unique string that identifies your package. Follow + the [Maven naming conventions](https://maven.apache.org/guides/mini/guide-naming-conventions.html). + - `DartifactId`: The name of the `JAR`, appended to the end of the `DgroupId`. + - `DarchetypeArtifactId`: The archetype used to create the initial structure of + the project. + - `DinteractiveMode`: Create the project using batch mode (optional). -After running the command, you should see the following message, indicating that -your project has been set up successfully: +This message indicates that the project was set up successfully: ```shell ... @@ -82,33 +79,33 @@ your project has been set up successfully: [INFO] ------------------------------------------------------------------------ ``` -You should see a new directory where you ran this command matching your -`DartifactId` parameter (in this case it should be `my-project`). +In the folder where you ran the command, a new directory should be displayed. +The directory name should match the `DartifactId` parameter, which in this case, +is `my-project`. -## Use Gradle to create a Java project +## Build a Java project with Gradle This section explains how to install Gradle and initialize a Java project. If you already use Gradle and know how to build your own packages, go to the -[next section](#add-the-package-registry-as-a-maven-remote). +[next section](#authenticate-to-the-package-registry-with-maven). ### Install Gradle -Installation is needed only if you want to create a new Gradle project. Follow +If you want to create a new Gradle project, you must install Gradle. Follow instructions at [gradle.org](https://gradle.org/install/) to download and install Gradle for your local development environment. -Verify you can use Gradle in your terminal by running: +In your terminal, verify you can use Gradle by running: ```shell gradle -version ``` -If you want to use an existing Gradle project, installation is not necessary. -Simply execute `gradlew` (on Linux) or `gradlew.bat` (on Windows) in the project -directory instead. +To use an existing Gradle project, in the project directory, +on Linux execute `gradlew`, or on Windows execute `gradlew.bat`. -You should see something similar to the below printed in the output: +The output should be similar to: ```plaintext ------------------------------------------------------------ @@ -127,88 +124,79 @@ OS: Windows 10 10.0 amd64 ### Create a Java project -Understanding how to create a full Java project in Gradle is outside the scope of this -guide, but you can follow the steps below to create a new project that can be +Follow these steps to create a Maven project that can be published to the GitLab Package Registry. -Start by opening your terminal and creating a directory where you would like to -store the project in your environment. From inside the directory, you can run -the following Maven command to initialize a new package: +1. Open your terminal and create a directory to store the project. +1. From this new directory, run this Maven command to initialize a new package: -```shell -gradle init -``` + ```shell + gradle init + ``` -The output should be + The output should be: -```plaintext -Select type of project to generate: - 1: basic - 2: application - 3: library - 4: Gradle plugin -Enter selection (default: basic) [1..4] -``` + ```plaintext + Select type of project to generate: + 1: basic + 2: application + 3: library + 4: Gradle plugin + Enter selection (default: basic) [1..4] + ``` -Enter `3` to create a new Library project. The output should be: +1. Enter `3` to create a new Library project. The output should be: -```plaintext -Select implementation language: - 1: C++ - 2: Groovy - 3: Java - 4: Kotlin - 5: Scala - 6: Swift -``` + ```plaintext + Select implementation language: + 1: C++ + 2: Groovy + 3: Java + 4: Kotlin + 5: Scala + 6: Swift + ``` -Enter `3` to create a new Java Library project. The output should be: +1. Enter `3` to create a new Java Library project. The output should be: -```plaintext -Select build script DSL: - 1: Groovy - 2: Kotlin -Enter selection (default: Groovy) [1..2] -``` + ```plaintext + Select build script DSL: + 1: Groovy + 2: Kotlin + Enter selection (default: Groovy) [1..2] + ``` -Choose `1` to create a new Java Library project which is described in Groovy DSL. The output should be: +1. Enter `1` to create a new Java Library project that is described in Groovy DSL. The output should be: -```plaintext -Select test framework: - 1: JUnit 4 - 2: TestNG - 3: Spock - 4: JUnit Jupiter -``` + ```plaintext + Select test framework: + 1: JUnit 4 + 2: TestNG + 3: Spock + 4: JUnit Jupiter + ``` -Choose `1` to initialize the project with JUnit 4 testing libraries. The output should be: +1. Enter `1` to initialize the project with JUnit 4 testing libraries. The output should be: -```plaintext -Project name (default: test): -``` + ```plaintext + Project name (default: test): + ``` -Enter a project name or hit enter to use the directory name as project name. +1. Enter a project name or press Enter to use the directory name as project name. -## Add the Package Registry as a Maven remote +## Authenticate to the Package Registry with Maven -The next step is to add the GitLab Package Registry as a Maven remote. If a -project is private or you want to upload Maven artifacts to GitLab, -credentials must be provided for authorization too. Support is available -for [personal access tokens](#authenticate-with-a-personal-access-token), -[CI job tokens](#authenticate-with-a-ci-job-token), and -[deploy tokens](../../project/deploy_tokens/index.md) only. Regular username/password -credentials do not work. +To authenticate to the Package Registry, you need either a personal access token or deploy token. -### Authenticate with a personal access token +- If you use a [personal access token](../../../user/profile/personal_access_tokens.md), set the scope to `api`. +- If you use a [deploy token](./../../project/deploy_tokens/index.md), set the scope to `read_package_registry`, `write_package_registry`, or both. -To authenticate with a [personal access token](../../profile/personal_access_tokens.md), -set the scope to `api` when creating one, and add it to your Maven or Gradle configuration -files. +### Authenticate with a personal access token in Maven -#### Authenticate with a personal access token in Maven +To use a personal access token, add this section to your +[`settings.xml`](https://maven.apache.org/settings.html) file. -Add a corresponding section to your -[`settings.xml`](https://maven.apache.org/settings.html) file: +The `name` must be `Private-Token`. ```xml <settings> @@ -228,45 +216,40 @@ Add a corresponding section to your </settings> ``` -#### Authenticate with a personal access token in Gradle +### Authenticate with a deploy token in Maven -Create a file `~/.gradle/gradle.properties` with the following content: +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) deploy token authentication in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. -```groovy -gitLabPrivateToken=REPLACE_WITH_YOUR_PERSONAL_ACCESS_TOKEN -``` +To use a deploy token, add this section to your +[`settings.xml`](https://maven.apache.org/settings.html) file. -Add a repositories section to your -[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) -file: +The `name` must be `Deploy-Token`. -```groovy -repositories { - maven { - url "https://<gitlab-url>/api/v4/groups/<group>/-/packages/maven" - name "GitLab" - credentials(HttpHeaderCredentials) { - name = 'Private-Token' - value = gitLabPrivateToken - } - authentication { - header(HttpHeaderAuthentication) - } - } -} +```xml +<settings> + <servers> + <server> + <id>gitlab-maven</id> + <configuration> + <httpHeaders> + <property> + <name>Deploy-Token</name> + <value>REPLACE_WITH_YOUR_DEPLOY_TOKEN</value> + </property> + </httpHeaders> + </configuration> + </server> + </servers> +</settings> ``` -You should now be able to upload Maven artifacts to your project. - -### Authenticate with a CI job token +### Authenticate with a CI job token in Maven -If you're using GitLab CI/CD, a CI job token can be used instead -of a personal access token. +To authenticate with a CI job token, add this section to your +[`settings.xml`](https://maven.apache.org/settings.html) file. -#### Authenticate with a CI job token in Maven - -To authenticate with a CI job token, add a corresponding section to your -[`settings.xml`](https://maven.apache.org/settings.html) file: +The `name` must be `Job-Token`. ```xml <settings> @@ -286,23 +269,35 @@ To authenticate with a CI job token, add a corresponding section to your </settings> ``` -You can read more on -[how to create Maven packages using GitLab CI/CD](#creating-maven-packages-with-gitlab-cicd). +Read more about [how to create Maven packages using GitLab CI/CD](#create-maven-packages-with-gitlab-cicd). + +## Authenticate to the Package Registry with Gradle -#### Authenticate with a CI job token in Gradle +To authenticate to the Package Registry, you need either a personal access token or deploy token. -To authenticate with a CI job token, add a repositories section to your +- If you use a [personal access token](../../../user/profile/personal_access_tokens.md), set the scope to `api`. +- If you use a [deploy token](./../../project/deploy_tokens/index.md), set the scope to `read_package_registry`, `write_package_registry`, or both. + +### Authenticate with a personal access token in Gradle + +Create a file `~/.gradle/gradle.properties` with the following content: + +```groovy +gitLabPrivateToken=REPLACE_WITH_YOUR_PERSONAL_ACCESS_TOKEN +``` + +Add a `repositories` section to your [`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) file: ```groovy repositories { maven { - url "https://<gitlab-url>/api/v4/groups/<group>/-/packages/maven" + url "https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven" name "GitLab" credentials(HttpHeaderCredentials) { - name = 'Job-Token' - value = System.getenv("CI_JOB_TOKEN") + name = 'Private-Token' + value = gitLabPrivateToken } authentication { header(HttpHeaderAuthentication) @@ -311,51 +306,42 @@ repositories { } ``` -### Authenticate with a deploy token - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. - -To authenticate with a [deploy token](./../../project/deploy_tokens/index.md), -set the scope to `api` when creating one, and add it to your Maven or Gradle configuration -files. - -#### Authenticate with a deploy token in Maven +### Authenticate with a deploy token in Gradle -Add a corresponding section to your -[`settings.xml`](https://maven.apache.org/settings.html) file: +To authenticate with a deploy token, add a `repositories` section to your +[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) +file: -```xml -<settings> - <servers> - <server> - <id>gitlab-maven</id> - <configuration> - <httpHeaders> - <property> - <name>Deploy-Token</name> - <value>REPLACE_WITH_YOUR_DEPLOY_TOKEN</value> - </property> - </httpHeaders> - </configuration> - </server> - </servers> -</settings> +```groovy +repositories { + maven { + url "https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = 'Deploy-Token' + value = '<deploy-token>' + } + authentication { + header(HttpHeaderAuthentication) + } + } +} ``` -#### Authenticate with a deploy token in Gradle +### Authenticate with a CI job token in Gradle -To authenticate with a deploy token, add a repositories section to your +To authenticate with a CI job token, add a `repositories` section to your [`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) file: ```groovy repositories { maven { - url "https://<gitlab-url>/api/v4/groups/<group>/-/packages/maven" + url "https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven" name "GitLab" credentials(HttpHeaderCredentials) { - name = 'Deploy-Token' - value = '<deploy-token>' + name = 'Job-Token' + value = System.getenv("CI_JOB_TOKEN") } authentication { header(HttpHeaderAuthentication) @@ -364,177 +350,162 @@ repositories { } ``` -## Configuring your project to use the GitLab Maven repository URL - -To download and upload packages from GitLab, you need a `repository` and -`distributionManagement` section in your `pom.xml` file. If you're following the -steps from above, then you must add the following information to your -`my-project/pom.xml` file. +## Use the GitLab endpoint for Maven packages -Depending on your workflow and the amount of Maven packages you have, there are -3 ways you can configure your project to use the GitLab endpoint for Maven packages: +To use the GitLab endpoint for Maven packages, choose an option: -- **Project level**: Useful when you have few Maven packages which are not under +- **Project-level**: Use when you have few Maven packages and they are not in the same GitLab group. -- **Group level**: Useful when you have many Maven packages under the same GitLab +- **Group-level**: Use when you have many Maven packages in the same GitLab group. -- **Instance level**: Useful when you have many Maven packages under different - GitLab groups or on their own namespace. +- **Instance-level**: Use when you have many Maven packages in different + GitLab groups or in their own namespace. + +The option you choose determines the settings you'll add to your `pom.xml` file. -NOTE: **Note:** -In all cases, you need a project specific URL for uploading a package in -the `distributionManagement` section. +In all cases, to publish a package, you need: -### Project level Maven endpoint +- A project-specific URL in the `distributionManagement` section. +- A `repository` and `distributionManagement` section. -The example below shows how the relevant `repository` section of your `pom.xml` -would look like in Maven: +### Project-level Maven endpoint + +The relevant `repository` section of your `pom.xml` +in Maven should look like this: ```xml <repositories> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> </repository> </repositories> <distributionManagement> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> </repository> <snapshotRepository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> </snapshotRepository> </distributionManagement> ``` -The corresponding section in Gradle would look like this: +The corresponding section in Gradle would be: ```groovy repositories { maven { - url "https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven" + url "https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven" name "GitLab" } } ``` -The `id` must be the same with what you -[defined in `settings.xml`](#add-the-package-registry-as-a-maven-remote). - -Replace `PROJECT_ID` with your project ID which can be found on the home page -of your project. +- The `id` is what you [defined in `settings.xml`](#authenticate-to-the-package-registry-with-maven). +- The `PROJECT_ID` is your project ID, which you can view on your project's home page. +- Replace `gitlab.example.com` with your domain name. +- For retrieving artifacts, use either the + [URL-encoded](../../../api/README.md#namespaced-path-encoding) path of the project + (like `group%2Fproject`) or the project's ID (like `42`). However, only the + project's ID can be used for publishing. -If you have a self-managed GitLab installation, replace `gitlab.com` with your -domain name. +### Group-level Maven endpoint -NOTE: **Note:** -For retrieving artifacts, you can use either the -[URL encoded](../../../api/README.md#namespaced-path-encoding) path of the project -(such as `group%2Fproject`) or the project's ID (such as `42`). However, only the -project's ID can be used for uploading. - -### Group level Maven endpoint - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8798) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8798) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. If you rely on many packages, it might be inefficient to include the `repository` section -with a unique URL for each package. Instead, you can use the group level endpoint for -all your Maven packages stored within one GitLab group. Only packages you have access to +with a unique URL for each package. Instead, you can use the group-level endpoint for +all the Maven packages stored within one GitLab group. Only packages you have access to are available for download. -The group level endpoint works with any package names, which means the you -have the flexibility of naming compared to [instance level endpoint](#instance-level-maven-endpoint). -However, GitLab does not guarantee the uniqueness of the package names within +The group-level endpoint works with any package names, so you +have more flexibility in naming, compared to the [instance-level endpoint](#instance-level-maven-endpoint). +However, GitLab does not guarantee the uniqueness of package names within the group. You can have two projects with the same package name and package version. As a result, GitLab serves whichever one is more recent. -The example below shows how the relevant `repository` section of your `pom.xml` -would look like. You still need a project specific URL for uploading a package in +This example shows the relevant `repository` section of your `pom.xml` file. +You still need a project-specific URL for publishing a package in the `distributionManagement` section: ```xml <repositories> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/groups/GROUP_ID/-/packages/maven</url> + <url>https://gitlab.example.com/api/v4/groups/GROUP_ID/-/packages/maven</url> </repository> </repositories> <distributionManagement> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> </repository> <snapshotRepository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> </snapshotRepository> </distributionManagement> ``` -For Gradle, the corresponding repositories section would look like: +For Gradle, the corresponding `repositories` section would look like: ```groovy repositories { maven { - url "https://gitlab.com/api/v4/groups/GROUP_ID/-/packages/maven" + url "https://gitlab.example.com/api/v4/groups/GROUP_ID/-/packages/maven" name "GitLab" } } ``` -The `id` must be the same with what you -[defined in `settings.xml`](#add-the-package-registry-as-a-maven-remote). - -Replace `my-group` with your group name and `PROJECT_ID` with your project ID -which can be found on the home page of your project. - -If you have a self-managed GitLab installation, replace `gitlab.com` with your -domain name. +- For the `id`, use what you [defined in `settings.xml`](#authenticate-to-the-package-registry-with-maven). +- For `my-group`, use your group name. +- For `PROJECT_ID`, use your project ID, which you can view on your project's home page. +- Replace `gitlab.example.com` with your domain name. +- For retrieving artifacts, use either the + [URL-encoded](../../../api/README.md#namespaced-path-encoding) path of the group + (like `group%2Fsubgroup`) or the group's ID (like `12`). -NOTE: **Note:** -For retrieving artifacts, you can use either the -[URL encoded](../../../api/README.md#namespaced-path-encoding) path of the group -(such as `group%2Fsubgroup`) or the group's ID (such as `12`). +### Instance-level Maven endpoint -### Instance level Maven endpoint - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8274) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8274) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) to GitLab Core in 13.3. If you rely on many packages, it might be inefficient to include the `repository` section -with a unique URL for each package. Instead, you can use the instance level endpoint for -all maven packages stored in GitLab and the packages you have access to are available +with a unique URL for each package. Instead, you can use the instance-level endpoint for +all Maven packages stored in GitLab. All packages you have access to are available for download. -Note that **only packages that have the same path as the project** are exposed via -the instance level endpoint. +**Only packages that have the same path as the project** are exposed by +the instance-level endpoint. -| Project | Package | Instance level endpoint available | +| Project | Package | Instance-level endpoint available | | ------- | ------- | --------------------------------- | | `foo/bar` | `foo/bar/1.0-SNAPSHOT` | Yes | | `gitlab-org/gitlab` | `foo/bar/1.0-SNAPSHOT` | No | | `gitlab-org/gitlab` | `gitlab-org/gitlab/1.0-SNAPSHOT` | Yes | -The example below shows how the relevant `repository` section of your `pom.xml` -would look like. You still need a project specific URL for uploading a package in -the `distributionManagement` section: +This example shows how relevant `repository` section of your `pom.xml`. +You still need a project-specific URL in the `distributionManagement` section. ```xml <repositories> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/packages/maven</url> + <url>https://gitlab.example.com/api/v4/packages/maven</url> </repository> </repositories> <distributionManagement> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> </repository> <snapshotRepository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> </snapshotRepository> </distributionManagement> ``` @@ -544,40 +515,35 @@ The corresponding repositories section in Gradle would look like: ```groovy repositories { maven { - url "https://gitlab.com/api/v4/packages/maven" + url "https://gitlab.example.com/api/v4/packages/maven" name "GitLab" } } ``` -The `id` must be the same with what you -[defined in `settings.xml`](#add-the-package-registry-as-a-maven-remote). - -Replace `PROJECT_ID` with your project ID which can be found on the home page -of your project. - -If you have a self-managed GitLab installation, replace `gitlab.com` with your -domain name. +- The `id` is what you [defined in `settings.xml`](#authenticate-to-the-package-registry-with-maven). +- The `PROJECT_ID` is your project ID, which you can view on your project's home page. +- Replace `gitlab.example.com` with your domain name. +- For retrieving artifacts, use either the + [URL-encoded](../../../api/README.md#namespaced-path-encoding) path of the project + (like `group%2Fproject`) or the project's ID (like `42`). However, only the + project's ID can be used for publishing. -NOTE: **Note:** -For retrieving artifacts, you can use either the -[URL encoded](../../../api/README.md#namespaced-path-encoding) path of the project -(such as `group%2Fproject`) or the project's ID (such as `42`). However, only the -project's ID can be used for uploading. +## Publish a package -## Uploading packages +After you have set up the [remote and authentication](#authenticate-to-the-package-registry-with-maven) +and [configured your project](#use-the-gitlab-endpoint-for-maven-packages), +publish a Maven artifact from your project. -Once you have set up the [remote and authentication](#add-the-package-registry-as-a-maven-remote) -and [configured your project](#configuring-your-project-to-use-the-gitlab-maven-repository-url), -test to upload a Maven artifact from a project of yours. +### Publish by using Maven -### Upload using Maven +To publish a package by using Maven: ```shell mvn deploy ``` -If the deploy is successful, you should see the build success message again: +If the deploy is successful, the build success message should be displayed: ```shell ... @@ -585,108 +551,112 @@ If the deploy is successful, you should see the build success message again: ... ``` -You should also see that the upload was uploaded to the correct registry: +The message should also show that the package was published to the correct location: ```shell -Uploading to gitlab-maven: https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.jar +Uploading to gitlab-maven: https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.jar ``` -### Upload using Gradle +### Publish by using Gradle -Add the Gradle plugin [`maven-publish`](https://docs.gradle.org/current/userguide/publishing_maven.html) to the plugins section: +To publish a package by using Gradle: -```groovy -plugins { - id 'java' - id 'maven-publish' -} -``` +1. Add the Gradle plugin [`maven-publish`](https://docs.gradle.org/current/userguide/publishing_maven.html) to the plugins section: -Add a `publishing` section: + ```groovy + plugins { + id 'java' + id 'maven-publish' + } + ``` -```groovy -publishing { - publications { - library(MavenPublication) { - from components.java - } - } - repositories { - maven { - url "https://gitlab.com/api/v4/projects/<PROJECT_ID>/packages/maven" - credentials(HttpHeaderCredentials) { - name = "Private-Token" - value = gitLabPrivateToken // the variable resides in ~/.gradle/gradle.properties - } - authentication { - header(HttpHeaderAuthentication) - } - } - } -} -``` +1. Add a `publishing` section: + + ```groovy + publishing { + publications { + library(MavenPublication) { + from components.java + } + } + repositories { + maven { + url "https://gitlab.example.com/api/v4/projects/<PROJECT_ID>/packages/maven" + credentials(HttpHeaderCredentials) { + name = "Private-Token" + value = gitLabPrivateToken // the variable resides in ~/.gradle/gradle.properties + } + authentication { + header(HttpHeaderAuthentication) + } + } + } + } + ``` -Replace `PROJECT_ID` with your project ID which can be found on the home page -of your project. +1. Replace `PROJECT_ID` with your project ID, which can be found on your project's home page. -Run the publish task: +1. Run the publish task: -```shell -gradle publish -``` + ```shell + gradle publish + ``` -You can then navigate to your project's **Packages & Registries** page and see the uploaded -artifacts or even delete them. +Now navigate to your project's **Packages & Registries** page and view the published artifacts. -## Installing a package +## Install a package -Installing a package from the GitLab Package Registry requires that you set up -the [remote and authentication](#add-the-package-registry-as-a-maven-remote) -as above. Once this is completed, there are two ways to install a package. +To install a package from the GitLab Package Registry, you must configure +the [remote and authenticate](#authenticate-to-the-package-registry-with-maven). +When this is completed, there are two ways to install a package. -### Install using Maven with `mvn install` +### Use Maven with `mvn install` -Add the dependency manually to your project `pom.xml` file. To add the example -created above, the XML would look like: +To install a package by using `mvn install`: -```xml -<dependency> - <groupId>com.mycompany.mydepartment</groupId> - <artifactId>my-project</artifactId> - <version>1.0-SNAPSHOT</version> -</dependency> -``` +1. Add the dependency manually to your project `pom.xml` file. + To add the example created earlier, the XML would be: -Then, inside your project, run the following: + ```xml + <dependency> + <groupId>com.mycompany.mydepartment</groupId> + <artifactId>my-project</artifactId> + <version>1.0-SNAPSHOT</version> + </dependency> + ``` -```shell -mvn install -``` +1. In your project, run the following: + + ```shell + mvn install + ``` -Provided everything is set up correctly, you should see the dependency -downloaded from the GitLab Package Registry: +The message should show that the package is downloading from the Package Registry: ```shell -Downloading from gitlab-maven: http://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom +Downloading from gitlab-maven: http://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom ``` -### Install using Maven with `mvn dependency:get` +### Use Maven with `mvn dependency:get` -The second way to install packages is to use the Maven commands directly. -Inside your project directory, run: +You can install packages by using the Maven commands directly. + +1. In your project directory, run: + + ```shell + mvn dependency:get -Dartifact=com.nickkipling.app:nick-test-app:1.1-SNAPSHOT + ``` + +The message should show that the package is downloading from the Package Registry: ```shell -mvn dependency:get -Dartifact=com.nickkipling.app:nick-test-app:1.1-SNAPSHOT +Downloading from gitlab-maven: http://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom ``` -You should see the same downloading message confirming that the project was -retrieved from the GitLab Package Registry. - TIP: **Tip:** -Both the XML block and Maven command are readily copy and pastable from the -Package details page, allowing for quick and easy installation. +In the GitLab UI, on the Package Registry page for Maven, you can view and copy these commands. -### Install using Gradle +### Use Gradle Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to `build.gradle` in the dependencies section: @@ -696,25 +666,25 @@ dependencies { } ``` -## Removing a package +## Remove a package -In the packages view of your project page, you can delete packages by clicking -the red trash icons or by clicking the **Delete** button on the package details -page. +For your project, go to **Packages & Registries > Package Registry**. -## Creating Maven packages with GitLab CI/CD +To remove a package, click the red trash icon or, from the package details, the **Delete** button. -Once you have your repository configured to use the GitLab Maven Repository, +## Create Maven packages with GitLab CI/CD + +After you have configured your repository to use the Package Repository for Maven, you can configure GitLab CI/CD to build new packages automatically. -### Creating Maven packages with GitLab CI/CD using Maven +### Create Maven packages with GitLab CI/CD by using Maven -The example below shows how to create a new package each time the `master` branch -is updated: +You can create a new package each time the `master` branch is updated. 1. Create a `ci_settings.xml` file that serves as Maven's `settings.xml` file. - Add the server section with the same ID you defined in your `pom.xml` file. - For example, in our case it's `gitlab-maven`: + +1. Add the `server` section with the same ID you defined in your `pom.xml` file. + For example, use `gitlab-maven` as the ID: ```xml <settings xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" @@ -735,30 +705,29 @@ is updated: </settings> ``` -1. Make sure your `pom.xml` file includes the following: +1. Make sure your `pom.xml` file includes the following. + You can either let Maven use the CI environment variables, as shown in this example, + or you can hard code your project's ID. ```xml <repositories> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/maven</url> </repository> </repositories> <distributionManagement> <repository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/maven</url> </repository> <snapshotRepository> <id>gitlab-maven</id> - <url>https://gitlab.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/maven</url> </snapshotRepository> </distributionManagement> ``` - TIP: **Tip:** - You can either let Maven utilize the CI environment variables or hardcode your project's ID. - 1. Add a `deploy` job to your `.gitlab-ci.yml` file: ```yaml @@ -773,16 +742,17 @@ is updated: 1. Push those files to your repository. The next time the `deploy` job runs, it copies `ci_settings.xml` to the -user's home location (in this case the user is `root` since it runs in a -Docker container), and Maven uses the configured CI -[environment variables](../../../ci/variables/README.md#predefined-environment-variables). +user's home location. In this example: + +- The user is `root`, because the job runs in a Docker container. +- Maven uses the configured CI [environment variables](../../../ci/variables/README.md#predefined-environment-variables). -### Creating Maven packages with GitLab CI/CD using Gradle +### Create Maven packages with GitLab CI/CD by using Gradle -The example below shows how to create a new package each time the `master` branch -is updated: +You can create a package each time the `master` branch +is updated. -1. Make sure you use the Job-Token authentication as described in ["Authenticating with a CI job token in Gradle"](#authenticate-with-a-ci-job-token-in-gradle). +1. Authenticate with [a CI job token in Gradle](#authenticate-with-a-ci-job-token-in-gradle). 1. Add a `deploy` job to your `.gitlab-ci.yml` file: @@ -795,11 +765,13 @@ is updated: - master ``` -1. Push those files to your repository. +1. Commit files to your repository. + +When the pipeline is successful, the package is created. ### Version validation -The version string is validated using the following regex. +The version string is validated by using the following regex. ```ruby \A(\.?[\w\+-]+\.?)+\z @@ -827,7 +799,7 @@ When you set these options, all network requests are logged and a large amount o ### Useful Maven command-line options There are some [Maven command-line options](https://maven.apache.org/ref/current/maven-embedder/cli.html) -that may be useful when performing tasks with GitLab CI/CD. +that you can use when performing tasks with GitLab CI/CD. - File transfer progress can make the CI logs hard to read. Option `-ntp,--no-transfer-progress` was added in @@ -835,7 +807,7 @@ that may be useful when performing tasks with GitLab CI/CD. Alternatively, look at `-B,--batch-mode` [or lower level logging changes.](https://stackoverflow.com/questions/21638697/disable-maven-download-progress-indication) -- Specify where to find the POM file (`-f,--file`): +- Specify where to find the `pom.xml` file (`-f,--file`): ```yaml package: @@ -852,11 +824,10 @@ that may be useful when performing tasks with GitLab CI/CD. - 'mvn -s settings/ci.xml package' ``` -### Verifying your Maven settings +### Verify your Maven settings -If you encounter issues within CI that relate to the `settings.xml` file, it might be useful -to add an additional script task or job to -[verify the effective settings](https://maven.apache.org/plugins/maven-help-plugin/effective-settings-mojo.html). +If you encounter issues within CI/CD that relate to the `settings.xml` file, try adding +an additional script task or job to [verify the effective settings](https://maven.apache.org/plugins/maven-help-plugin/effective-settings-mojo.html). The help plugin can also provide [system properties](https://maven.apache.org/plugins/maven-help-plugin/system-mojo.html), including environment variables: diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index ae9ca5b2333..4b93ab93475 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -31,7 +31,7 @@ authenticate with GitLab by using the `CI_JOB_TOKEN`. CI/CD templates, which you can use to get started, are in [this repo](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). -Learn more about [using CI/CD to build Maven packages](../maven_repository/index.md#creating-maven-packages-with-gitlab-cicd), [NPM packages](../npm_registry/index.md#publishing-a-package-with-cicd), [Composer packages](../composer_repository/index.md#publish-a-composer-package-by-using-cicd), [NuGet Packages](../nuget_repository/index.md#publishing-a-nuget-package-with-cicd), [Conan Packages](../conan_repository/index.md#publish-a-conan-package-by-using-cicd), [PyPI packages](../pypi_repository/index.md#using-gitlab-ci-with-pypi-packages), and [generic packages](../generic_packages/index.md#publish-a-generic-package-by-using-cicd). +Learn more about [using CI/CD to build Maven packages](../maven_repository/index.md#create-maven-packages-with-gitlab-cicd), [NPM packages](../npm_registry/index.md#publishing-a-package-with-cicd), [Composer packages](../composer_repository/index.md#publish-a-composer-package-by-using-cicd), [NuGet Packages](../nuget_repository/index.md#publishing-a-nuget-package-with-cicd), [Conan Packages](../conan_repository/index.md#publish-a-conan-package-by-using-cicd), [PyPI packages](../pypi_repository/index.md#using-gitlab-ci-with-pypi-packages), and [generic packages](../generic_packages/index.md#publish-a-generic-package-by-using-cicd). If you use CI/CD to build a package, extended activity information is displayed when you view the package details: diff --git a/doc/user/packages/workflows/project_registry.md b/doc/user/packages/workflows/project_registry.md index 24437e6dfac..19190c29457 100644 --- a/doc/user/packages/workflows/project_registry.md +++ b/doc/user/packages/workflows/project_registry.md @@ -75,8 +75,8 @@ you do this, you can push your NPM package to your project using `npm publish`, If you are using Maven, this involves updating your `pom.xml` file with distribution sections, including the appropriate URL for your project, as described in the [GitLab Maven Repository documentation](../maven_repository/index.md#project-level-maven-endpoint). -Then, you need to add a `settings.xml` file and [include your access token](../maven_repository/index.md#authenticate-with-a-personal-access-token). -Now you can [deploy Maven packages](../maven_repository/index.md#uploading-packages) to your project. +Then, you need to add a `settings.xml` file and [include your access token](../maven_repository/index.md#authenticate-with-a-personal-access-token-in-maven). +Now you can [deploy Maven packages](../maven_repository/index.md#publish-a-package) to your project. #### Conan diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index 61fce7a2c1b..f3894818f28 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -11,7 +11,7 @@ module API helpers do def client @client ||= if Feature.enabled?(:remove_legacy_github_client) - Gitlab::GithubImport::Client.new(params[:personal_access_token]) + Gitlab::GithubImport::Client.new(params[:personal_access_token], host: params[:github_hostname]) else Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options) end @@ -22,7 +22,7 @@ module API end def client_options - {} + { host: params[:github_hostname] } end def provider @@ -43,6 +43,7 @@ module API requires :repo_id, type: Integer, desc: 'GitHub repository ID' optional :new_name, type: String, desc: 'New repo name' requires :target_namespace, type: String, desc: 'Namespace to import repo into' + optional :github_hostname, type: String, desc: 'Custom GitHub enterprise hostname' end post 'import/github' do result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 9a7c406d981..c3cc15e10f7 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -6,10 +6,13 @@ module Gitlab [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head'] end - def self.new_client_for(project, token: nil, parallel: true) + def self.new_client_for(project, token: nil, host: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) - - Client.new(token_to_use, parallel: parallel) + Client.new( + token_to_use, + host: host.presence || self.formatted_import_url(project), + parallel: parallel + ) end # Returns the ID of the ghost user. @@ -18,5 +21,17 @@ module Gitlab Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, User.select(:id).ghost.id) end + + # Get formatted GitHub import URL. If github.com is in the import URL, this will return nil and octokit will use the default github.com API URL + def self.formatted_import_url(project) + url = URI.parse(project.import_url) + + unless url.host == 'github.com' + url.user = nil + url.password = nil + url.path = "/api/v3" + url.to_s + end + end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 22803c5cd71..29a61f7c20a 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -31,6 +31,8 @@ module Gitlab # token - The GitHub API token to use. # + # host - The GitHub hostname. If nil, github.com will be used. + # # per_page - The number of objects that should be displayed per page. # # parallel - When set to true hitting the rate limit will result in a @@ -39,11 +41,13 @@ module Gitlab # this value to `true` for parallel importing is crucial as # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. - def initialize(token, per_page: 100, parallel: true) + def initialize(token, host: nil, per_page: 100, parallel: true) + @host = host @octokit = ::Octokit::Client.new( access_token: token, per_page: per_page, - api_endpoint: api_endpoint + api_endpoint: api_endpoint, + web_endpoint: web_endpoint ) @octokit.connection_options[:ssl] = { verify: verify_ssl } @@ -181,7 +185,11 @@ module Gitlab end def api_endpoint - custom_api_endpoint || default_api_endpoint + @host || custom_api_endpoint || default_api_endpoint + end + + def web_endpoint + @host || custom_api_endpoint || ::Octokit::Default.web_endpoint end def custom_api_endpoint diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 6a181caf65d..cb6b2017208 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -25,10 +25,11 @@ module Gitlab # project - The project to import the data into. # token - The token to use for the GitHub API. - def initialize(project, token: nil) + # host - The GitHub hostname. If nil, github.com will be used. + def initialize(project, token: nil, host: nil) @project = project @client = GithubImport - .new_client_for(project, token: token, parallel: false) + .new_client_for(project, token: token, host: host, parallel: false) end def execute diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index f7eaafeb446..4482610523e 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -24,6 +24,7 @@ module Gitlab @api ||= ::Octokit::Client.new( access_token: access_token, api_endpoint: api_endpoint, + web_endpoint: web_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -85,6 +86,10 @@ module Gitlab end end + def web_endpoint + host.presence || ::Octokit::Default.web_endpoint + end + def config Gitlab::Auth::OAuth::Provider.config_for('github') end diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 5f6ab42d0d2..20e84bacd55 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -299,6 +299,32 @@ RSpec.describe Gitlab::GithubImport::Client do end end + describe '#web_endpoint' do + let(:client) { described_class.new('foo') } + + context 'without a custom endpoint configured in Omniauth' do + it 'returns the default web endpoint' do + expect(client) + .to receive(:custom_api_endpoint) + .and_return(nil) + + expect(client.web_endpoint).to eq('https://github.com') + end + end + + context 'with a custom endpoint configured in Omniauth' do + it 'returns the custom endpoint' do + endpoint = 'https://github.kittens.com' + + expect(client) + .to receive(:custom_api_endpoint) + .and_return(endpoint) + + expect(client.web_endpoint).to eq(endpoint) + end + end + end + describe '#custom_api_endpoint' do let(:client) { described_class.new('foo') } diff --git a/spec/lib/gitlab/github_import/sequential_importer_spec.rb b/spec/lib/gitlab/github_import/sequential_importer_spec.rb index fe13fcd2568..a5e89049ed9 100644 --- a/spec/lib/gitlab/github_import/sequential_importer_spec.rb +++ b/spec/lib/gitlab/github_import/sequential_importer_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::GithubImport::SequentialImporter do describe '#execute' do it 'imports a project in sequence' do repository = double(:repository) - project = double(:project, id: 1, repository: repository) + project = double(:project, id: 1, repository: repository, import_url: 'http://t0ken@github.another-domain.com/repo-org/repo.git') importer = described_class.new(project, token: 'foo') expect_next_instance_of(Gitlab::GithubImport::Importer::RepositoryImporter) do |instance| diff --git a/spec/lib/gitlab/github_import_spec.rb b/spec/lib/gitlab/github_import_spec.rb index 1a690b81d2b..3129da64809 100644 --- a/spec/lib/gitlab/github_import_spec.rb +++ b/spec/lib/gitlab/github_import_spec.rb @@ -3,13 +3,13 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport do - let(:project) { double(:project) } + context 'github.com' do + let(:project) { double(:project, import_url: 'http://t0ken@github.com/user/repo.git') } - describe '.new_client_for' do it 'returns a new Client with a custom token' do expect(described_class::Client) .to receive(:new) - .with('123', parallel: true) + .with('123', host: nil, parallel: true) described_class.new_client_for(project, token: '123') end @@ -23,18 +23,57 @@ RSpec.describe Gitlab::GithubImport do expect(described_class::Client) .to receive(:new) - .with('123', parallel: true) + .with('123', host: nil, parallel: true) described_class.new_client_for(project) end + + it 'returns the ID of the ghost user', :clean_gitlab_redis_cache do + expect(described_class.ghost_user_id).to eq(User.ghost.id) + end + + it 'caches the ghost user ID', :clean_gitlab_redis_cache do + expect(Gitlab::Cache::Import::Caching) + .to receive(:write) + .once + .and_call_original + + 2.times do + described_class.ghost_user_id + end + end end - describe '.ghost_user_id', :clean_gitlab_redis_cache do - it 'returns the ID of the ghost user' do + context 'GitHub Enterprise' do + let(:project) { double(:project, import_url: 'http://t0ken@github.another-domain.com/repo-org/repo.git') } + + it 'returns a new Client with a custom token' do + expect(described_class::Client) + .to receive(:new) + .with('123', host: 'http://github.another-domain.com/api/v3', parallel: true) + + described_class.new_client_for(project, token: '123') + end + + it 'returns a new Client with a token stored in the import data' do + import_data = double(:import_data, credentials: { user: '123' }) + + expect(project) + .to receive(:import_data) + .and_return(import_data) + + expect(described_class::Client) + .to receive(:new) + .with('123', host: 'http://github.another-domain.com/api/v3', parallel: true) + + described_class.new_client_for(project) + end + + it 'returns the ID of the ghost user', :clean_gitlab_redis_cache do expect(described_class.ghost_user_id).to eq(User.ghost.id) end - it 'caches the ghost user ID' do + it 'caches the ghost user ID', :clean_gitlab_redis_cache do expect(Gitlab::Cache::Import::Caching) .to receive(:write) .once @@ -44,5 +83,9 @@ RSpec.describe Gitlab::GithubImport do described_class.ghost_user_id end end + + it 'formats the import url' do + expect(described_class.formatted_import_url(project)).to eq('http://github.another-domain.com/api/v3') + end end end diff --git a/spec/requests/api/import_github_spec.rb b/spec/requests/api/import_github_spec.rb index bbfb17fe753..5bbcb0c1950 100644 --- a/spec/requests/api/import_github_spec.rb +++ b/spec/requests/api/import_github_spec.rb @@ -57,6 +57,22 @@ RSpec.describe API::ImportGithub do expect(json_response['name']).to eq(project.name) end + it 'returns 201 response when the project is imported successfully from GHE' do + allow(Gitlab::LegacyGithubImport::ProjectCreator) + .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) + .and_return(double(execute: project)) + + post api("/import/github", user), params: { + target_namespace: user.namespace_path, + personal_access_token: token, + repo_id: non_existing_record_id, + github_hostname: "https://github.somecompany.com/" + } + expect(response).to have_gitlab_http_status(:created) + expect(json_response).to be_a Hash + expect(json_response['name']).to eq(project.name) + end + it 'returns 422 response when user can not create projects in the chosen namespace' do other_namespace = create(:group, name: 'other_namespace') diff --git a/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb b/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb index 09d64fe50bd..8727756ce50 100644 --- a/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb @@ -26,7 +26,7 @@ RSpec.describe Gitlab::GithubImport::ReschedulingMethods do end context 'with an existing project' do - let(:project) { create(:project) } + let(:project) { create(:project, import_url: 'https://t0ken@github.com/repo/repo.git') } it 'notifies any waiters upon successfully importing the data' do expect(worker) diff --git a/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb b/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb index b7635748498..03e875bcb87 100644 --- a/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb @@ -9,6 +9,8 @@ RSpec.describe Gitlab::GithubImport::StageMethods do end describe '#perform' do + let(:project) { create(:project, import_url: 'https://t0ken@github.com/repo/repo.git') } + it 'returns if no project could be found' do expect(worker).not_to receive(:try_import) |