From 4e7ceeb1cb4bce13f14ad151788548b521b9d319 Mon Sep 17 00:00:00 2001 From: Jihad Khawaja Date: Sun, 27 Jul 2025 15:42:14 +0300 Subject: [PATCH] init --- .github/workflows/compile.yml | 249 ++++++++++++------ LLama/LLamaSharp.Runtime.targets | 27 ++ LLama/Native/Load/NativeLibraryUtils.cs | 4 +- LLama/Native/Load/NativeLibraryWithAvx.cs | 2 +- .../build/LLamaSharp.Backend.Cpu.nuspec | 7 + 5 files changed, 205 insertions(+), 84 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index ccc013653..7309aa499 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -80,11 +80,12 @@ jobs: with: path: ./build/bin/libggml-cpu.so name: ggml-cpu-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so - if-no-files-found: error - - uses: actions/upload-artifact@v4 + if-no-files-found: error + - name: Upload Llava + uses: actions/upload-artifact@v4 with: - path: ./build/bin/libmtmd.so - name: mtmd-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so + path: ./build/bin/libllava_shared.so + name: llava-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so if-no-files-found: error compile-musl: @@ -145,11 +146,12 @@ jobs: with: path: ./build/bin/libggml-cpu.so name: ggml-cpu-bin-musl-${{ matrix.build }}-x64.so - if-no-files-found: error - - uses: actions/upload-artifact@v4 + if-no-files-found: error + - name: Upload Llava + uses: actions/upload-artifact@v4 with: - path: ./build/bin/libmtmd.so - name: mtmd-bin-musl-${{ matrix.build }}-x64.so + path: ./build/bin/libllava_shared.so + name: llava-bin-musl-${{ matrix.build }}-x64.so if-no-files-found: error compile-windows: @@ -160,12 +162,74 @@ jobs: include: - build: 'noavx' defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF' + arch: 'x64' - build: 'avx2' defines: '' + arch: 'x64' - build: 'avx' defines: '-DGGML_AVX2=OFF' + arch: 'x64' - build: 'avx512' defines: '-DGGML_AVX512=ON -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON' + arch: 'x64' + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + repository: ggerganov/llama.cpp + fetch-depth: 0 + ref: '${{ github.event.inputs.llama_cpp_commit }}' + + - name: Build + id: cmake_build + run: | + mkdir build + cd build + cmake .. ${{ env.COMMON_DEFINE }} ${{ matrix.defines }} + cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS} + tree /f + + - name: Upload artifacts (llama) + uses: actions/upload-artifact@v4 + with: + path: .\build\bin\Release\llama.dll + name: llama-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll + if-no-files-found: error + - name: Upload artifacts (ggml) + uses: actions/upload-artifact@v4 + with: + path: .\build\bin\Release\ggml.dll + name: ggml-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll + if-no-files-found: error + - name: Upload artifacts (ggml-base) + uses: actions/upload-artifact@v4 + with: + path: .\build\bin\Release\ggml-base.dll + name: ggml-base-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll + if-no-files-found: error + - name: Upload artifacts (ggml-cpu) + uses: actions/upload-artifact@v4 + with: + path: .\build\bin\Release\ggml-cpu.dll + name: ggml-cpu-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll + if-no-files-found: error + + - name: Upload artifacts (llava) + uses: actions/upload-artifact@v4 + with: + path: .\build\bin\Release\llava_shared.dll + name: llava-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll + if-no-files-found: error + + compile-windows-arm64: + name: Compile (Windows ARM64) + strategy: + fail-fast: true + matrix: + include: + - build: 'arm64' + defines: '-DCMAKE_GENERATOR_PLATFORM=ARM64 -DGGML_NATIVE=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF' + arch: 'arm64' runs-on: windows-latest steps: - uses: actions/checkout@v4 @@ -187,31 +251,32 @@ jobs: uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\llama.dll - name: llama-bin-win-${{ matrix.build }}-x64.dll + name: llama-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll if-no-files-found: error - name: Upload artifacts (ggml) uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\ggml.dll - name: ggml-bin-win-${{ matrix.build }}-x64.dll + name: ggml-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll if-no-files-found: error - name: Upload artifacts (ggml-base) uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\ggml-base.dll - name: ggml-base-bin-win-${{ matrix.build }}-x64.dll + name: ggml-base-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll if-no-files-found: error - name: Upload artifacts (ggml-cpu) uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\ggml-cpu.dll - name: ggml-cpu-bin-win-${{ matrix.build }}-x64.dll + name: ggml-cpu-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll if-no-files-found: error - - name: Upload artifacts (mtmd) + + - name: Upload artifacts (llava) uses: actions/upload-artifact@v4 with: - path: .\build\bin\Release\mtmd.dll - name: mtmd-bin-win-${{ matrix.build }}-x64.dll + path: .\build\bin\Release\llava_shared.dll + name: llava-bin-win-${{ matrix.build }}-${{ matrix.arch }}.dll if-no-files-found: error compile-vulkan: @@ -294,13 +359,13 @@ jobs: with: path: .\build\bin\Release\ggml-vulkan.dll name: ggml-vulkan-bin-win-vulkan-x64.dll - if-no-files-found: error - - name: Upload mtmd artifacts (Windows) + if-no-files-found: error + - name: Upload llava artifacts (Windows) if: ${{ matrix.os == 'windows-latest' }} uses: actions/upload-artifact@v4 with: - path: .\build\bin\Release\mtmd.dll - name: mtmd-bin-win-vulkan-x64.dll + path: .\build\bin\Release\llava_shared.dll + name: llava-bin-win-vulkan-x64.dll if-no-files-found: error - name: Upload llama artifacts (Linux) if: ${{ matrix.os == 'ubuntu-22.04' }} @@ -329,13 +394,13 @@ jobs: with: path: ./build/bin/libggml-vulkan.so name: ggml-vulkan-bin-linux-vulkan-x64.so - if-no-files-found: error - - name: Upload mtmd artifacts (Linux) + if-no-files-found: error + - name: Upload llava artifacts (Linux) if: ${{ matrix.os == 'ubuntu-22.04' }} uses: actions/upload-artifact@v4 with: - path: ./build/bin/libmtmd.so - name: mtmd-bin-linux-vulkan-x64.so + path: ./build/bin/libllava_shared.so + name: llava-bin-linux-vulkan-x64.so if-no-files-found: error compile-cublas: @@ -343,8 +408,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, windows-2022] - cuda: ['12.4.0'] + os: [ubuntu-22.04, windows-2019] + cuda: ['12.2.0', '11.7.1'] runs-on: ${{ matrix.os }} steps: - name: Clone @@ -382,39 +447,39 @@ jobs: ls -R - name: Upload artifacts (Windows) - if: ${{ matrix.os == 'windows-2022' }} + if: ${{ matrix.os == 'windows-2019' }} uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\llama.dll name: llama-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll if-no-files-found: error - name: Upload artifacts (ggml) - if: ${{ matrix.os == 'windows-2022' }} + if: ${{ matrix.os == 'windows-2019' }} uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\ggml.dll name: ggml-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll if-no-files-found: error - name: Upload artifacts (ggml-base) - if: ${{ matrix.os == 'windows-2022' }} + if: ${{ matrix.os == 'windows-2019' }} uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\ggml-base.dll name: ggml-base-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll if-no-files-found: error - name: Upload artifacts (ggml-cuda) - if: ${{ matrix.os == 'windows-2022' }} + if: ${{ matrix.os == 'windows-2019' }} uses: actions/upload-artifact@v4 with: path: .\build\bin\Release\ggml-cuda.dll name: ggml-cuda-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll - if-no-files-found: error - - name: Upload mtmd artifacts (Windows) - if: ${{ matrix.os == 'windows-2022' }} + if-no-files-found: error + - name: Upload llava artifacts (Windows) + if: ${{ matrix.os == 'windows-2019' }} uses: actions/upload-artifact@v4 with: - path: .\build\bin\Release\mtmd.dll - name: mtmd-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll + path: .\build\bin\Release\llava_shared.dll + name: llava-bin-win-cublas-cu${{ matrix.cuda }}-x64.dll if-no-files-found: error - name: Upload artifacts (Linux) @@ -444,13 +509,13 @@ jobs: with: path: ./build/bin/libggml-cuda.so name: ggml-cuda-bin-linux-cublas-cu${{ matrix.cuda }}-x64.so - if-no-files-found: error - - name: Upload mtmd artifacts (Linux) + if-no-files-found: error + - name: Upload llava artifacts (Linux) if: ${{ matrix.os == 'ubuntu-22.04' }} uses: actions/upload-artifact@v4 with: - path: ./build/bin/libmtmd.so - name: mtmd-bin-linux-cublas-cu${{ matrix.cuda }}-x64.so + path: ./build/bin/libllava_shared.so + name: llava-bin-linux-cublas-cu${{ matrix.cuda }}-x64.so if-no-files-found: error compile-macos: @@ -520,12 +585,12 @@ jobs: with: path: ./build/bin/libllama.dylib name: llama-bin-osx-${{ matrix.build }}.dylib - if-no-files-found: error - - name: Upload mtmd + if-no-files-found: error + - name: Upload Llava uses: actions/upload-artifact@v4 with: - path: ./build/bin/libmtmd.dylib - name: mtmd-bin-osx-${{ matrix.build }}.dylib + path: ./build/bin/libllava_shared.dylib + name: llava-bin-osx-${{ matrix.build }}.dylib if-no-files-found: error - name: Upload Metal if: ${{ matrix.build == 'arm64' }} @@ -588,12 +653,12 @@ jobs: with: path: ./build/bin/libggml-cpu.so name: ggml-cpu-bin-android-${{ matrix.build }}.so - if-no-files-found: error - - name: Upload mtmd + if-no-files-found: error + - name: Upload Llava uses: actions/upload-artifact@v4 with: - path: ./build/bin/libmtmd.so - name: mtmd-bin-android-${{ matrix.build }}.so + path: ./build/bin/libllava_shared.so + name: llava-bin-android-${{ matrix.build }}.so build-deps: runs-on: ubuntu-latest @@ -603,6 +668,7 @@ jobs: "compile-linux", "compile-musl", "compile-windows", + "compile-windows-arm64", "compile-vulkan", "compile-cublas", "compile-macos", @@ -617,89 +683,96 @@ jobs: - name: Rearrange Files run: | # Make all directories at once - mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu12.4.0,vulkan,android-arm64-v8a,android-x86,android-x86_64} + mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64,win-arm64} # Linux cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so deps/noavx/libggml.so cp artifacts/ggml-base-bin-linux-noavx-x64.so/libggml-base.so deps/noavx/libggml-base.so cp artifacts/ggml-cpu-bin-linux-noavx-x64.so/libggml-cpu.so deps/noavx/libggml-cpu.so - cp artifacts/llama-bin-linux-noavx-x64.so/libllama.so deps/noavx/libllama.so - cp artifacts/mtmd-bin-linux-noavx-x64.so/libmtmd.so deps/noavx/libmtmd.so + cp artifacts/llama-bin-linux-noavx-x64.so/libllama.so deps/noavx/libllama.so + cp artifacts/llava-bin-linux-noavx-x64.so/libllava_shared.so deps/noavx/libllava_shared.so cp artifacts/ggml-bin-linux-avx-x64.so/libggml.so deps/avx/libggml.so cp artifacts/ggml-base-bin-linux-avx-x64.so/libggml-base.so deps/avx/libggml-base.so cp artifacts/ggml-cpu-bin-linux-avx-x64.so/libggml-cpu.so deps/avx/libggml-cpu.so - cp artifacts/llama-bin-linux-avx-x64.so/libllama.so deps/avx/libllama.so - cp artifacts/mtmd-bin-linux-avx-x64.so/libmtmd.so deps/avx/libmtmd.so + cp artifacts/llama-bin-linux-avx-x64.so/libllama.so deps/avx/libllama.so + cp artifacts/llava-bin-linux-avx-x64.so/libllava_shared.so deps/avx/libllava_shared.so cp artifacts/ggml-bin-linux-avx2-x64.so/libggml.so deps/avx2/libggml.so cp artifacts/ggml-base-bin-linux-avx2-x64.so/libggml-base.so deps/avx2/libggml-base.so cp artifacts/ggml-cpu-bin-linux-avx2-x64.so/libggml-cpu.so deps/avx2/libggml-cpu.so - cp artifacts/llama-bin-linux-avx2-x64.so/libllama.so deps/avx2/libllama.so - cp artifacts/mtmd-bin-linux-avx2-x64.so/libmtmd.so deps/avx2/libmtmd.so + cp artifacts/llama-bin-linux-avx2-x64.so/libllama.so deps/avx2/libllama.so + cp artifacts/llava-bin-linux-avx2-x64.so/libllava_shared.so deps/avx2/libllava_shared.so cp artifacts/ggml-bin-linux-avx512-x64.so/libggml.so deps/avx512/libggml.so cp artifacts/ggml-base-bin-linux-avx512-x64.so/libggml-base.so deps/avx512/libggml-base.so cp artifacts/ggml-cpu-bin-linux-avx512-x64.so/libggml-cpu.so deps/avx512/libggml-cpu.so - cp artifacts/llama-bin-linux-avx512-x64.so/libllama.so deps/avx512/libllama.so - cp artifacts/mtmd-bin-linux-avx512-x64.so/libmtmd.so deps/avx512/libmtmd.so + cp artifacts/llama-bin-linux-avx512-x64.so/libllama.so deps/avx512/libllama.so + cp artifacts/llava-bin-linux-avx512-x64.so/libllava_shared.so deps/avx512/libllava_shared.so # Arm64 cp artifacts/ggml-bin-linux-aarch64-arm64.so/libggml.so deps/linux-arm64/libggml.so cp artifacts/ggml-base-bin-linux-aarch64-arm64.so/libggml-base.so deps/linux-arm64/libggml-base.so cp artifacts/ggml-cpu-bin-linux-aarch64-arm64.so/libggml-cpu.so deps/linux-arm64/libggml-cpu.so cp artifacts/llama-bin-linux-aarch64-arm64.so/libllama.so deps/linux-arm64/libllama.so - cp artifacts/mtmd-bin-linux-aarch64-arm64.so/libmtmd.so deps/linux-arm64/libmtmd.so + cp artifacts/llava-bin-linux-aarch64-arm64.so/libllava_shared.so deps/linux-arm64/libllava_shared.so # Musl cp artifacts/ggml-bin-musl-noavx-x64.so/libggml.so deps/musl-noavx/libggml.so cp artifacts/ggml-base-bin-musl-noavx-x64.so/libggml-base.so deps/musl-noavx/libggml-base.so cp artifacts/ggml-cpu-bin-musl-noavx-x64.so/libggml-cpu.so deps/musl-noavx/libggml-cpu.so cp artifacts/llama-bin-musl-noavx-x64.so/libllama.so deps/musl-noavx/libllama.so - cp artifacts/mtmd-bin-musl-noavx-x64.so/libmtmd.so deps/musl-noavx/libmtmd.so + cp artifacts/llava-bin-musl-noavx-x64.so/libllava_shared.so deps/musl-noavx/libllava_shared.so cp artifacts/ggml-bin-musl-avx-x64.so/libggml.so deps/musl-avx/libggml.so cp artifacts/ggml-base-bin-musl-avx-x64.so/libggml-base.so deps/musl-avx/libggml-base.so cp artifacts/ggml-cpu-bin-musl-avx-x64.so/libggml-cpu.so deps/musl-avx/libggml-cpu.so cp artifacts/llama-bin-musl-avx-x64.so/libllama.so deps/musl-avx/libllama.so - cp artifacts/mtmd-bin-musl-avx-x64.so/libmtmd.so deps/musl-avx/libmtmd.so + cp artifacts/llava-bin-musl-avx-x64.so/libllava_shared.so deps/musl-avx/libllava_shared.so cp artifacts/ggml-bin-musl-avx2-x64.so/libggml.so deps/musl-avx2/libggml.so cp artifacts/ggml-base-bin-musl-avx2-x64.so/libggml-base.so deps/musl-avx2/libggml-base.so cp artifacts/ggml-cpu-bin-musl-avx2-x64.so/libggml-cpu.so deps/musl-avx2/libggml-cpu.so cp artifacts/llama-bin-musl-avx2-x64.so/libllama.so deps/musl-avx2/libllama.so - cp artifacts/mtmd-bin-musl-avx2-x64.so/libmtmd.so deps/musl-avx2/libmtmd.so + cp artifacts/llava-bin-musl-avx2-x64.so/libllava_shared.so deps/musl-avx2/libllava_shared.so cp artifacts/ggml-bin-musl-avx512-x64.so/libggml.so deps/musl-avx512/libggml.so cp artifacts/ggml-base-bin-musl-avx512-x64.so/libggml-base.so deps/musl-avx512/libggml-base.so cp artifacts/ggml-cpu-bin-musl-avx512-x64.so/libggml-cpu.so deps/musl-avx512/libggml-cpu.so cp artifacts/llama-bin-musl-avx512-x64.so/libllama.so deps/musl-avx512/libllama.so - cp artifacts/mtmd-bin-musl-avx512-x64.so/libmtmd.so deps/musl-avx512/libmtmd.so + cp artifacts/llava-bin-musl-avx512-x64.so/libllava_shared.so deps/musl-avx512/libllava_shared.so # Windows cp artifacts/ggml-bin-win-noavx-x64.dll/ggml.dll deps/noavx/ggml.dll cp artifacts/ggml-base-bin-win-noavx-x64.dll/ggml-base.dll deps/noavx/ggml-base.dll cp artifacts/ggml-cpu-bin-win-noavx-x64.dll/ggml-cpu.dll deps/noavx/ggml-cpu.dll cp artifacts/llama-bin-win-noavx-x64.dll/llama.dll deps/noavx/llama.dll - cp artifacts/mtmd-bin-win-noavx-x64.dll/mtmd.dll deps/noavx/mtmd.dll + cp artifacts/llava-bin-win-noavx-x64.dll/llava_shared.dll deps/noavx/llava_shared.dll cp artifacts/ggml-bin-win-avx-x64.dll/ggml.dll deps/avx/ggml.dll cp artifacts/ggml-base-bin-win-avx-x64.dll/ggml-base.dll deps/avx/ggml-base.dll cp artifacts/ggml-cpu-bin-win-avx-x64.dll/ggml-cpu.dll deps/avx/ggml-cpu.dll cp artifacts/llama-bin-win-avx-x64.dll/llama.dll deps/avx/llama.dll - cp artifacts/mtmd-bin-win-avx-x64.dll/mtmd.dll deps/avx/mtmd.dll + cp artifacts/llava-bin-win-avx-x64.dll/llava_shared.dll deps/avx/llava_shared.dll cp artifacts/ggml-bin-win-avx2-x64.dll/ggml.dll deps/avx2/ggml.dll cp artifacts/ggml-base-bin-win-avx2-x64.dll/ggml-base.dll deps/avx2/ggml-base.dll cp artifacts/ggml-cpu-bin-win-avx2-x64.dll/ggml-cpu.dll deps/avx2/ggml-cpu.dll cp artifacts/llama-bin-win-avx2-x64.dll/llama.dll deps/avx2/llama.dll - cp artifacts/mtmd-bin-win-avx2-x64.dll/mtmd.dll deps/avx2/mtmd.dll + cp artifacts/llava-bin-win-avx2-x64.dll/llava_shared.dll deps/avx2/llava_shared.dll cp artifacts/ggml-bin-win-avx512-x64.dll/ggml.dll deps/avx512/ggml.dll cp artifacts/ggml-base-bin-win-avx512-x64.dll/ggml-base.dll deps/avx512/ggml-base.dll cp artifacts/ggml-cpu-bin-win-avx512-x64.dll/ggml-cpu.dll deps/avx512/ggml-cpu.dll cp artifacts/llama-bin-win-avx512-x64.dll/llama.dll deps/avx512/llama.dll - cp artifacts/mtmd-bin-win-avx512-x64.dll/mtmd.dll deps/avx512/mtmd.dll + cp artifacts/llava-bin-win-avx512-x64.dll/llava_shared.dll deps/avx512/llava_shared.dll + + # Windows ARM64 + cp artifacts/ggml-bin-win-arm64-arm64.dll/ggml.dll deps/win-arm64/ggml.dll + cp artifacts/ggml-base-bin-win-arm64-arm64.dll/ggml-base.dll deps/win-arm64/ggml-base.dll + cp artifacts/ggml-cpu-bin-win-arm64-arm64.dll/ggml-cpu.dll deps/win-arm64/ggml-cpu.dll + cp artifacts/llama-bin-win-arm64-arm64.dll/llama.dll deps/win-arm64/llama.dll + cp artifacts/llava-bin-win-arm64-arm64.dll/llava_shared.dll deps/win-arm64/llava_shared.dll # MacOS cp artifacts/ggml-bin-osx-arm64.dylib/libggml.dylib deps/osx-arm64/libggml.dylib @@ -708,7 +781,7 @@ jobs: cp artifacts/ggml-blas-bin-osx-arm64.dylib/libggml-blas.dylib deps/osx-arm64/libggml-blas.dylib cp artifacts/ggml-metal-bin-osx-arm64.dylib/libggml-metal.dylib deps/osx-arm64/libggml-metal.dylib cp artifacts/llama-bin-osx-arm64.dylib/libllama.dylib deps/osx-arm64/libllama.dylib - cp artifacts/mtmd-bin-osx-arm64.dylib/libmtmd.dylib deps/osx-arm64/libmtmd.dylib + cp artifacts/llava-bin-osx-arm64.dylib/libllava_shared.dylib deps/osx-arm64/libllava_shared.dylib cp artifacts/ggml-metal.metal/ggml-metal.metal deps/osx-arm64/ggml-metal.metal cp artifacts/ggml-bin-osx-x64.dylib/libggml.dylib deps/osx-x64/libggml.dylib @@ -716,55 +789,67 @@ jobs: cp artifacts/ggml-cpu-bin-osx-x64.dylib/libggml-cpu.dylib deps/osx-x64/libggml-cpu.dylib cp artifacts/ggml-blas-bin-osx-x64.dylib/libggml-blas.dylib deps/osx-x64/libggml-blas.dylib cp artifacts/llama-bin-osx-x64.dylib/libllama.dylib deps/osx-x64/libllama.dylib - cp artifacts/mtmd-bin-osx-x64.dylib/libmtmd.dylib deps/osx-x64/libmtmd.dylib + cp artifacts/llava-bin-osx-x64.dylib/libllava_shared.dylib deps/osx-x64/libllava_shared.dylib cp artifacts/ggml-bin-osx-x64-rosetta2.dylib/libggml.dylib deps/osx-x64-rosetta2/libggml.dylib cp artifacts/ggml-base-bin-osx-x64-rosetta2.dylib/libggml-base.dylib deps/osx-x64-rosetta2/libggml-base.dylib cp artifacts/ggml-cpu-bin-osx-x64-rosetta2.dylib/libggml-cpu.dylib deps/osx-x64-rosetta2/libggml-cpu.dylib cp artifacts/ggml-blas-bin-osx-x64-rosetta2.dylib/libggml-blas.dylib deps/osx-x64-rosetta2/libggml-blas.dylib cp artifacts/llama-bin-osx-x64-rosetta2.dylib/libllama.dylib deps/osx-x64-rosetta2/libllama.dylib - cp artifacts/mtmd-bin-osx-x64-rosetta2.dylib/libmtmd.dylib deps/osx-x64-rosetta2/libmtmd.dylib + cp artifacts/llava-bin-osx-x64-rosetta2.dylib/libllava_shared.dylib deps/osx-x64-rosetta2/libllava_shared.dylib # Android cp artifacts/ggml-bin-android-arm64-v8a.so/libggml.so deps/android-arm64-v8a/libggml.so cp artifacts/ggml-base-bin-android-arm64-v8a.so/libggml-base.so deps/android-arm64-v8a/libggml-base.so cp artifacts/ggml-cpu-bin-android-arm64-v8a.so/libggml-cpu.so deps/android-arm64-v8a/libggml-cpu.so cp artifacts/llama-bin-android-arm64-v8a.so/libllama.so deps/android-arm64-v8a/libllama.so - cp artifacts/mtmd-bin-android-arm64-v8a.so/libmtmd.so deps/android-arm64-v8a/libmtmd.so + cp artifacts/llava-bin-android-arm64-v8a.so/libllava_shared.so deps/android-arm64-v8a/libllava_shared.so cp artifacts/ggml-bin-android-x86_64.so/libggml.so deps/android-x86_64/libggml.so cp artifacts/ggml-base-bin-android-x86_64.so/libggml-base.so deps/android-x86_64/libggml-base.so cp artifacts/ggml-cpu-bin-android-x86_64.so/libggml-cpu.so deps/android-x86_64/libggml-cpu.so cp artifacts/llama-bin-android-x86_64.so/libllama.so deps/android-x86_64/libllama.so - cp artifacts/mtmd-bin-android-x86_64.so/libmtmd.so deps/android-x86_64/libmtmd.so + cp artifacts/llava-bin-android-x86_64.so/libllava_shared.so deps/android-x86_64/libllava_shared.so # Windows CUDA - cp artifacts/ggml-bin-win-cublas-cu12.4.0-x64.dll/ggml.dll deps/cu12.4.0/ggml.dll - cp artifacts/ggml-base-bin-win-cublas-cu12.4.0-x64.dll/ggml-base.dll deps/cu12.4.0/ggml-base.dll - cp artifacts/ggml-cuda-bin-win-cublas-cu12.4.0-x64.dll/ggml-cuda.dll deps/cu12.4.0/ggml-cuda.dll - cp artifacts/llama-bin-win-cublas-cu12.4.0-x64.dll/llama.dll deps/cu12.4.0/llama.dll - cp artifacts/mtmd-bin-win-cublas-cu12.4.0-x64.dll/mtmd.dll deps/cu12.4.0/mtmd.dll + cp artifacts/ggml-bin-win-cublas-cu11.7.1-x64.dll/ggml.dll deps/cu11.7.1/ggml.dll + cp artifacts/ggml-base-bin-win-cublas-cu11.7.1-x64.dll/ggml-base.dll deps/cu11.7.1/ggml-base.dll + cp artifacts/ggml-cuda-bin-win-cublas-cu11.7.1-x64.dll/ggml-cuda.dll deps/cu11.7.1/ggml-cuda.dll + cp artifacts/llama-bin-win-cublas-cu11.7.1-x64.dll/llama.dll deps/cu11.7.1/llama.dll + cp artifacts/llava-bin-win-cublas-cu11.7.1-x64.dll/llava_shared.dll deps/cu11.7.1/llava_shared.dll + + cp artifacts/ggml-bin-win-cublas-cu12.2.0-x64.dll/ggml.dll deps/cu12.2.0/ggml.dll + cp artifacts/ggml-base-bin-win-cublas-cu12.2.0-x64.dll/ggml-base.dll deps/cu12.2.0/ggml-base.dll + cp artifacts/ggml-cuda-bin-win-cublas-cu12.2.0-x64.dll/ggml-cuda.dll deps/cu12.2.0/ggml-cuda.dll + cp artifacts/llama-bin-win-cublas-cu12.2.0-x64.dll/llama.dll deps/cu12.2.0/llama.dll + cp artifacts/llava-bin-win-cublas-cu12.2.0-x64.dll/llava_shared.dll deps/cu12.2.0/llava_shared.dll # Linux CUDA - cp artifacts/ggml-bin-linux-cublas-cu12.4.0-x64.so/libggml.so deps/cu12.4.0/libggml.so - cp artifacts/ggml-base-bin-linux-cublas-cu12.4.0-x64.so/libggml-base.so deps/cu12.4.0/libggml-base.so - cp artifacts/ggml-cuda-bin-linux-cublas-cu12.4.0-x64.so/libggml-cuda.so deps/cu12.4.0/libggml-cuda.so - cp artifacts/llama-bin-linux-cublas-cu12.4.0-x64.so/libllama.so deps/cu12.4.0/libllama.so - cp artifacts/mtmd-bin-linux-cublas-cu12.4.0-x64.so/libmtmd.so deps/cu12.4.0/libmtmd.so + cp artifacts/ggml-bin-linux-cublas-cu11.7.1-x64.so/libggml.so deps/cu11.7.1/libggml.so + cp artifacts/ggml-base-bin-linux-cublas-cu11.7.1-x64.so/libggml-base.so deps/cu11.7.1/libggml-base.so + cp artifacts/ggml-cuda-bin-linux-cublas-cu11.7.1-x64.so/libggml-cuda.so deps/cu11.7.1/libggml-cuda.so + cp artifacts/llama-bin-linux-cublas-cu11.7.1-x64.so/libllama.so deps/cu11.7.1/libllama.so + cp artifacts/llava-bin-linux-cublas-cu11.7.1-x64.so/libllava_shared.so deps/cu11.7.1/libllava_shared.so + + cp artifacts/ggml-bin-linux-cublas-cu12.2.0-x64.so/libggml.so deps/cu12.2.0/libggml.so + cp artifacts/ggml-base-bin-linux-cublas-cu12.2.0-x64.so/libggml-base.so deps/cu12.2.0/libggml-base.so + cp artifacts/ggml-cuda-bin-linux-cublas-cu12.2.0-x64.so/libggml-cuda.so deps/cu12.2.0/libggml-cuda.so + cp artifacts/llama-bin-linux-cublas-cu12.2.0-x64.so/libllama.so deps/cu12.2.0/libllama.so + cp artifacts/llava-bin-linux-cublas-cu12.2.0-x64.so/libllava_shared.so deps/cu12.2.0/libllava_shared.so # Windows Vulkan cp artifacts/ggml-bin-win-vulkan-x64.dll/ggml.dll deps/vulkan/ggml.dll cp artifacts/ggml-base-bin-win-vulkan-x64.dll/ggml-base.dll deps/vulkan/ggml-base.dll cp artifacts/ggml-vulkan-bin-win-vulkan-x64.dll/ggml-vulkan.dll deps/vulkan/ggml-vulkan.dll cp artifacts/llama-bin-win-vulkan-x64.dll/llama.dll deps/vulkan/llama.dll - cp artifacts/mtmd-bin-win-vulkan-x64.dll/mtmd.dll deps/vulkan/mtmd.dll + cp artifacts/llava-bin-win-vulkan-x64.dll/llava_shared.dll deps/vulkan/llava_shared.dll # Linux Vulkan cp artifacts/ggml-bin-linux-vulkan-x64.so/libggml.so deps/vulkan/libggml.so cp artifacts/ggml-base-bin-linux-vulkan-x64.so/libggml-base.so deps/vulkan/libggml-base.so cp artifacts/ggml-vulkan-bin-linux-vulkan-x64.so/libggml-vulkan.so deps/vulkan/libggml-vulkan.so cp artifacts/llama-bin-linux-vulkan-x64.so/libllama.so deps/vulkan/libllama.so - cp artifacts/mtmd-bin-linux-vulkan-x64.so/libmtmd.so deps/vulkan/libmtmd.so + cp artifacts/llava-bin-linux-vulkan-x64.so/libllava_shared.so deps/vulkan/libllava_shared.so - name: Upload artifacts uses: actions/upload-artifact@v4 @@ -778,6 +863,6 @@ jobs: with: name: | llama-* - mtmd-* + llava-* *.metal ggml-* diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets index 89caa042a..e4fb7c89a 100644 --- a/LLama/LLamaSharp.Runtime.targets +++ b/LLama/LLamaSharp.Runtime.targets @@ -112,6 +112,29 @@ + + + PreserveNewest + runtimes/win-arm64/native/llama.dll + + + PreserveNewest + runtimes/win-arm64/native/ggml.dll + + + PreserveNewest + runtimes/win-arm64/native/ggml-base.dll + + + PreserveNewest + runtimes/win-arm64/native/ggml-cpu.dll + + + PreserveNewest + runtimes/win-arm64/native/llava_shared.dll + + + PreserveNewest runtimes/linux-x64/native/noavx/libllama.so @@ -418,6 +441,10 @@ PreserveNewest runtimes/win-x64/native/vulkan/libmtmd.dll + + PreserveNewest + runtimes/win-arm64/native/libmtmd.dll + PreserveNewest diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs index 9f6457cd1..9ec996a20 100644 --- a/LLama/Native/Load/NativeLibraryUtils.cs +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -219,7 +219,9 @@ public static void GetPlatformPathParts(OSPlatform platform, out string os, out { if (platform == OSPlatform.Windows) { - os = "win-x64"; + os = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported + ? "win-arm64" + : "win-x64"; fileExtension = ".dll"; libPrefix = ""; return; diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs index e6cbd86f3..3296fac0f 100644 --- a/LLama/Native/Load/NativeLibraryWithAvx.cs +++ b/LLama/Native/Load/NativeLibraryWithAvx.cs @@ -50,7 +50,7 @@ public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaL private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback) { NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); - if (os != "linux-arm64"){ + if (os != "linux-arm64" && os != "win-arm64"){ var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel); if (!string.IsNullOrEmpty(avxStr)) avxStr += "/"; diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec index aeef403eb..db2693270 100644 --- a/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec +++ b/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec @@ -42,6 +42,13 @@ + + + + + + +