fix: optimize packages upload memory (#39366)

Bound the memory used when parsing uploaded packages, so instances with
low memory limits survive non-standard packages.

1. Metadata files are read through a size-limited reader, archives
themselves stay unbounded so large payloads keep working
2. zstd decoding caps the window at 128 MiB, the zstd CLI default
3. RPM headers go through a first-party reader with rpmlib's size and
array count limits that allocates only what the upload contains, until
https://github.com/sassoftware/go-rpmutils/pull/44 is released. Signing
still verifies the payload digest, and its signature header matches
rpmsign's byte for byte
4. Swift and NuGet read the zip entry they size-checked, a duplicate
name could stand in for it before
5. Terraform states check `LIMIT_SIZE_TERRAFORM_STATE` before parsing,
NuGet symbol PDBs share a 256 MiB budget
6. Oversized container configs are rejected instead of stored without
metadata, invalid Vagrant boxes answer 400 instead of 500

Signed-off-by: techknowlogick <git@tklk.dev>
Co-authored-by: techknowlogick <git@tklk.dev>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: techknowlogick <techknowlogick@gitea.com>
This commit is contained in:
afishcalledwander
2026-10-17 06:10:04 +00:00
committed by GitHub
co-authored by techknowlogick silverwind techknowlogick
parent c1ae4ec29e
commit 42b1f95261
33 changed files with 719 additions and 189 deletions
-15
View File
File diff suppressed because one or more lines are too long
-3
View File
@@ -76,7 +76,6 @@ require (
github.com/redis/go-redis/v9 v9.23.0
github.com/robfig/cron/v3 v3.0.1
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
github.com/sassoftware/go-rpmutils v0.4.0
github.com/sergi/go-diff v1.4.0
github.com/stretchr/testify v1.12.1
github.com/syndtr/goleveldb v1.0.0
@@ -109,7 +108,6 @@ require (
require (
cloud.google.com/go/compute/metadata v0.9.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/DataDog/zstd v1.5.7 // indirect
github.com/RoaringBitmap/roaring/v2 v2.18.2 // indirect
github.com/STARRY-S/zip v0.2.3 // indirect
github.com/andybalholm/brotli v1.2.1 // indirect
@@ -220,7 +218,6 @@ require (
github.com/stangelandcl/ppmd v0.1.1 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
-6
View File
@@ -34,8 +34,6 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/Necoro/html2text v0.0.0-20250804200300-7bf1ce1c7347 h1:3JhDl+JysaO8nhNU1XMaw35VSGjV4IEQAefaG4Lyok4=
github.com/Necoro/html2text v0.0.0-20250804200300-7bf1ce1c7347/go.mod h1:2ErI0aycD43Ufr6CFK5lT/NrHGmoZuVbn1nlPThw69o=
github.com/ProtonMail/go-crypto v1.5.2 h1:cucYnvqcY7UOXVD//mSyjeaPY0SSN3v5cDkYPxumINk=
@@ -509,8 +507,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg=
github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=
github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
@@ -562,8 +558,6 @@ github.com/wneessen/go-mail v0.8.1 h1:tVcncj02/QySVFw3zr/kXOzZcuFQqBNT6K+Rbgm/pc
github.com/wneessen/go-mail v0.8.1/go.mod h1:dWZ61zadzCIyvB4y1/YzC5O7MrbbzBfPkARmbosdf8w=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yohcop/openid-go v1.0.1 h1:DPRd3iPO5F6O5zX2e62XpVAbPT6wV51cuucH0z9g3js=
+4 -2
View File
@@ -16,8 +16,8 @@ import (
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"gitea.dev/modules/zstd"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
)
@@ -34,6 +34,8 @@ const (
RepositoryVersion = "_repository"
AnyArch = "any"
maxPKGINFOSize = 1024 * 1024
)
var (
@@ -178,7 +180,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
func ParsePackageInfo(r io.Reader) (*Package, error) {
p := &Package{}
s := bufio.NewScanner(r)
s := bufio.NewScanner(packages.NewLimitedReader(r, maxPKGINFOSize))
for s.Scan() {
line := s.Text()
+4
View File
@@ -10,6 +10,7 @@ import (
"regexp"
"gitea.dev/modules/json"
"gitea.dev/modules/packages"
"gitea.dev/modules/validation"
"github.com/hashicorp/go-version"
@@ -67,6 +68,9 @@ func ParsePackage(r io.Reader) (*Package, error) {
if err := binary.Read(r, binary.LittleEndian, &size); err != nil {
return nil, err
}
if size > packages.MaxMetadataSize {
return nil, packages.ErrContentTooLarge
}
p, err := parsePackage(io.LimitReader(r, int64(size)))
if err != nil {
+2 -1
View File
@@ -11,6 +11,7 @@ import (
"strings"
"gitea.dev/modules/json"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
)
@@ -103,7 +104,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
// ParseChefMetadata parses a metadata.json file to retrieve the metadata of a Chef package
func ParseChefMetadata(r io.Reader) (*Package, error) {
var cm chefMetadata
if err := json.NewDecoder(r).Decode(&cm); err != nil {
if err := json.NewDecoder(packages.NewLimitedReader(r, packages.MaxMetadataSize)).Decode(&cm); err != nil {
return nil, err
}
+3 -2
View File
@@ -11,6 +11,7 @@ import (
"strings"
"gitea.dev/modules/json"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
"gitea.dev/modules/zstd"
@@ -147,7 +148,7 @@ func parsePackageTar(r io.Reader) (*Package, error) {
}
if hdr.Name == "info/index.json" {
if err := json.NewDecoder(tr).Decode(&i); err != nil {
if err := json.NewDecoder(packages.NewLimitedReader(tr, packages.MaxMetadataSize)).Decode(&i); err != nil {
return nil, err
}
@@ -163,7 +164,7 @@ func parsePackageTar(r io.Reader) (*Package, error) {
break // stop loop if both files were found
}
} else if hdr.Name == "info/about.json" {
if err := json.NewDecoder(tr).Decode(&a); err != nil {
if err := json.NewDecoder(packages.NewLimitedReader(tr, packages.MaxMetadataSize)).Decode(&a); err != nil {
return nil, err
}
+6 -1
View File
@@ -4,11 +4,13 @@
package container
import (
"errors"
"fmt"
"io"
"strings"
"gitea.dev/modules/json"
"gitea.dev/modules/packages"
"gitea.dev/modules/packages/container/helm"
"gitea.dev/modules/validation"
@@ -83,8 +85,11 @@ func IsMediaTypeImageIndex(mt string) bool {
return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json")
}
const maxImageConfigSize = 8 * 1024 * 1024
// ParseImageConfig parses the metadata of an image config
func ParseImageConfig(mediaType string, r io.Reader) (*Metadata, error) {
r = packages.NewLimitedReader(r, maxImageConfigSize)
if strings.EqualFold(mediaType, helm.ConfigMediaType) {
return parseHelmConfig(r)
}
@@ -93,7 +98,7 @@ func ParseImageConfig(mediaType string, r io.Reader) (*Metadata, error) {
// FIXME: this fallback is not right, we should strictly check the media type in the future
metadata, err := parseOCIImageConfig(r)
if err != nil {
if !IsMediaTypeImageManifest(mediaType) {
if !IsMediaTypeImageManifest(mediaType) && !errors.Is(err, packages.ErrContentTooLarge) {
return &Metadata{Platform: "unknown/unknown"}, nil
}
return nil, err
@@ -7,6 +7,7 @@ import (
"strings"
"testing"
"gitea.dev/modules/packages"
"gitea.dev/modules/packages/container/helm"
oci "github.com/opencontainers/image-spec/specs-go/v1"
@@ -65,4 +66,7 @@ func TestParseImageConfig(t *testing.T) {
metadata, err = ParseImageConfig("anything-unknown", strings.NewReader(""))
require.NoError(t, err)
assert.Equal(t, &Metadata{Platform: "unknown/unknown"}, metadata)
_, err = ParseImageConfig(oci.MediaTypeImageConfig, strings.NewReader(`{"history": [{"created_by": "`+strings.Repeat("a", maxImageConfigSize)+`"}]}`))
assert.ErrorIs(t, err, packages.ErrContentTooLarge)
}
+4 -3
View File
@@ -13,6 +13,7 @@ import (
"regexp"
"strings"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
)
@@ -151,7 +152,7 @@ func ParseDescription(r io.Reader) (*Package, error) {
Metadata: &Metadata{},
}
scanner := bufio.NewScanner(r)
scanner := bufio.NewScanner(packages.NewLimitedReader(r, packages.MaxMetadataSize))
var b strings.Builder
for scanner.Scan() {
@@ -173,11 +174,11 @@ func ParseDescription(r io.Reader) (*Package, error) {
b.WriteString(line)
}
if err := setField(p, b.String()); err != nil {
if err := scanner.Err(); err != nil {
return nil, err
}
if err := scanner.Err(); err != nil {
if err := setField(p, b.String()); err != nil {
return nil, err
}
+1
View File
@@ -65,6 +65,7 @@ func CreateHashedBufferFromReaderWithSize(r io.Reader, maxMemorySize int) (*Hash
_, err = io.Copy(b, r)
if err != nil {
_ = b.Close()
return nil, err
}
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"io"
"strings"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
@@ -106,7 +107,7 @@ func ParseChartArchive(r io.Reader) (*Metadata, error) {
// ParseChartFile parses a Chart.yaml file to retrieve the metadata of a Helm chart
func ParseChartFile(r io.Reader) (*Metadata, error) {
var metadata *Metadata
if err := yaml.NewDecoder(r).Decode(&metadata); err != nil {
if err := yaml.NewDecoder(packages.NewLimitedReader(r, packages.MaxMetadataSize)).Decode(&metadata); err != nil {
return nil, err
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package packages
import (
"io"
"gitea.dev/modules/util"
)
var ErrContentTooLarge = util.NewInvalidArgumentErrorf("content exceeds the size limit")
const MaxMetadataSize = 32 * 1024 * 1024
type limitedReader struct {
io.LimitedReader
}
// NewLimitedReader returns a reader that fails with ErrContentTooLarge once it reads a byte past limit
func NewLimitedReader(r io.Reader, limit int64) io.Reader {
return &limitedReader{io.LimitedReader{R: r, N: limit + 1}}
}
func (l *limitedReader) Read(p []byte) (int, error) {
n, err := l.LimitedReader.Read(p)
if l.N == 0 {
return max(n-1, 0), ErrContentTooLarge
}
return n, err
}
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package packages
import (
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLimitedReader(t *testing.T) {
data, err := io.ReadAll(NewLimitedReader(strings.NewReader("abc"), 3))
assert.NoError(t, err)
assert.Equal(t, "abc", string(data))
r := NewLimitedReader(strings.NewReader("abcd"), 3)
data, err = io.ReadAll(r)
assert.ErrorIs(t, err, ErrContentTooLarge)
assert.Equal(t, "abc", string(data))
n, err := r.Read(make([]byte, 8))
assert.Zero(t, n)
assert.ErrorIs(t, err, ErrContentTooLarge)
}
+1 -1
View File
@@ -151,7 +151,7 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) {
if file.UncompressedSize64 > maxNuspecFileSize {
return nil, ErrNuspecFileTooLarge
}
f, err := archive.Open(file.Name)
f, err := file.Open()
if err != nil {
return nil, err
}
+12 -2
View File
@@ -6,6 +6,8 @@ package nuget
import (
"archive/zip"
"bytes"
"maps"
"slices"
"testing"
"github.com/stretchr/testify/assert"
@@ -82,9 +84,9 @@ func TestParsePackageMetaData(t *testing.T) {
createArchive := func(files map[string]string) []byte {
var buf bytes.Buffer
archive := zip.NewWriter(&buf)
for name, content := range files {
for _, name := range slices.Sorted(maps.Keys(files)) {
w, _ := archive.Create(name)
w.Write([]byte(content))
w.Write([]byte(files[name]))
}
archive.Close()
return buf.Bytes()
@@ -147,6 +149,14 @@ func TestParsePackageMetaData(t *testing.T) {
assert.Empty(t, np.Metadata.Readme)
})
t.Run("IgnoresNuspecShadowingRootNuspec", func(t *testing.T) {
data := createArchive(map[string]string{"/package.nuspec": "", "package.nuspec": nuspecContent})
np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data)))
assert.NoError(t, err)
assert.Equal(t, id, np.ID)
})
t.Run("Dependency Package", func(t *testing.T) {
data := createArchive(map[string]string{
"package.nuspec": nuspecContent,
+5 -1
View File
@@ -24,6 +24,8 @@ var (
ErrMissingPdbStream = util.NewInvalidArgumentErrorf("missing PDB stream")
)
const maxPdbTotalSize = 256 * 1024 * 1024
type PortablePdb struct {
Name string
ID string
@@ -46,6 +48,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) {
}
var pdbs PortablePdbList
remaining := int64(maxPdbTotalSize)
err = func() error {
for _, file := range archive.File {
@@ -63,13 +66,14 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) {
return err
}
buf, err := packages.CreateHashedBufferFromReader(f)
buf, err := packages.CreateHashedBufferFromReader(packages.NewLimitedReader(f, remaining))
_ = f.Close()
if err != nil {
return err
}
remaining -= buf.Size()
id, err := ParseDebugHeaderID(buf)
if err != nil {
+259
View File
@@ -0,0 +1,259 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package rpm
import (
"bytes"
"crypto/sha1"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"io"
"slices"
"strings"
"gitea.dev/modules/util"
)
// https://rpm-software-management.github.io/rpm/manual/format_v4.html
// TODO: replace this reader and SignPackage with go-rpmutils once https://github.com/sassoftware/go-rpmutils/pull/44 is released
var ErrInvalidPackage = util.NewInvalidArgumentErrorf("invalid RPM package")
const (
leadSize = 96
leadMagic = 0xedabeedb
headerMagic = 0x8eade801
headerIntroSize = 16
indexEntrySize = 16
maxHeaderEntries = 0xffff // rpmlib's hdrblobVerifyLengths limits
maxHeaderData = 0x0fffffff
maxHeaderArray = 0x000fffff
maxSignatureEntries = 32
maxSignatureData = 64 * 1024 * 1024
maxRegionTag = 63
)
const (
typeNull = iota
typeChar
typeInt8
typeInt16
typeInt32
typeInt64
typeString
typeBin
typeStringArray
typeI18NString
)
var typeSizes = map[uint32]int{typeNull: 0, typeChar: 1, typeInt8: 1, typeInt16: 2, typeInt32: 4, typeInt64: 8, typeBin: 1}
const (
tagName = 1000
tagVersion = 1001
tagRelease = 1002
tagEpoch = 1003
tagSummary = 1004
tagDescription = 1005
tagBuildTime = 1006
tagBuildHost = 1007
tagSize = 1009
tagVendor = 1011
tagLicense = 1014
tagPackager = 1015
tagGroup = 1016
tagURL = 1020
tagArch = 1022
tagFileModes = 1030
tagFileMTimes = 1034
tagFileFlags = 1037
tagSourceRpm = 1044
tagProvideName = 1047
tagRequireFlags = 1048
tagRequireName = 1049
tagRequireVersion = 1050
tagConflictFlags = 1053
tagConflictName = 1054
tagConflictVersion = 1055
tagChangelogTime = 1080
tagChangelogName = 1081
tagChangelogText = 1082
tagObsoleteName = 1090
tagProvideFlags = 1112
tagProvideVersion = 1113
tagObsoleteFlags = 1114
tagObsoleteVersion = 1115
tagDirIndexes = 1116
tagBaseNames = 1117
tagDirNames = 1118
tagPayloadDigest = 5092
tagPayloadDigestAlgo = 5093
sigTagRegion = 62
sigTagDSA = 267
sigTagRSA = 268
sigTagSHA1 = 269
sigTagSHA256 = 273
sigTagPGP = 1002
sigTagMD5 = 1004
sigTagGPG = 1005
sigTagPayloadSize = 1007
sigTagReservedSpace = 1008
)
type headerEntry struct {
typ, count uint32
data []byte
}
type header struct {
raw []byte
entries map[uint32]headerEntry
}
// readHeader validates the index like rpmlib's hdrblobVerifyInfo, so data is only allocated as it is read
func readHeader(r io.Reader, maxEntries, maxData uint32) (*header, error) {
intro := make([]byte, headerIntroSize)
if _, err := io.ReadFull(r, intro); err != nil {
return nil, err
}
indexCount, dataSize := binary.BigEndian.Uint32(intro[8:]), binary.BigEndian.Uint32(intro[12:])
if binary.BigEndian.Uint32(intro) != headerMagic || indexCount > maxEntries || dataSize > maxData {
return nil, ErrInvalidPackage
}
size := headerIntroSize + int64(indexCount)*indexEntrySize + int64(dataSize)
buf := bytes.NewBuffer(intro)
if _, err := io.CopyN(buf, r, size-headerIntroSize); err != nil {
return nil, err
}
h := &header{raw: buf.Bytes(), entries: make(map[uint32]headerEntry, indexCount)}
index := h.raw[headerIntroSize : headerIntroSize+indexCount*indexEntrySize]
store := h.raw[len(index)+headerIntroSize:]
end := uint32(0)
for i := range indexCount {
entry := index[i*indexEntrySize:]
tag, typ := binary.BigEndian.Uint32(entry), binary.BigEndian.Uint32(entry[4:])
offset, count := binary.BigEndian.Uint32(entry[8:]), binary.BigEndian.Uint32(entry[12:])
if i == 0 && tag <= maxRegionTag {
continue
}
if offset < end || offset > dataSize || (typ != typeBin && count > maxHeaderArray) {
return nil, ErrInvalidPackage
}
n, ok := dataLength(typ, count, store[offset:])
if !ok {
return nil, ErrInvalidPackage
}
end = offset + uint32(n)
h.entries[tag] = headerEntry{typ: typ, count: count, data: store[offset:end]}
}
return h, nil
}
func dataLength(typ, count uint32, data []byte) (int, bool) {
switch typ {
case typeString, typeStringArray, typeI18NString:
if int64(count) > int64(len(data)) {
return 0, false
}
n := 0
for range count {
i := bytes.IndexByte(data[n:], 0)
if i < 0 {
return 0, false
}
n += i + 1
}
return n, true
}
size, ok := typeSizes[typ]
n := int64(size) * int64(count)
return int(n), ok && n <= int64(len(data))
}
func readHeaders(r io.Reader) (lead []byte, sig, hdr *header, err error) {
defer func() {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
err = ErrInvalidPackage
}
}()
lead = make([]byte, leadSize)
if _, err := io.ReadFull(r, lead); err != nil {
return nil, nil, nil, err
}
if binary.BigEndian.Uint32(lead) != leadMagic {
return nil, nil, nil, ErrInvalidPackage
}
if sig, err = readHeader(r, maxSignatureEntries, maxSignatureData); err != nil {
return nil, nil, nil, err
}
if _, err := io.CopyN(io.Discard, r, int64(signaturePadding(len(sig.raw)))); err != nil {
return nil, nil, nil, err
}
if hdr, err = readHeader(r, maxHeaderEntries, maxHeaderData); err != nil {
return nil, nil, nil, err
}
if digest := sig.getString(sigTagSHA256); digest != "" {
if sum := sha256.Sum256(hdr.raw); hex.EncodeToString(sum[:]) != digest {
return nil, nil, nil, ErrInvalidPackage
}
} else if digest := sig.getString(sigTagSHA1); digest != "" {
if sum := sha1.Sum(hdr.raw); hex.EncodeToString(sum[:]) != digest {
return nil, nil, nil, ErrInvalidPackage
}
}
return lead, sig, hdr, nil
}
func signaturePadding(size int) int {
return (8 - size%8) % 8
}
func (h *header) getStrings(tag uint32) []string {
entry := h.entries[tag]
if entry.count == 0 || (entry.typ != typeString && entry.typ != typeStringArray && entry.typ != typeI18NString) {
return nil
}
return strings.Split(string(entry.data[:len(entry.data)-1]), "\x00")
}
func (h *header) getString(tag uint32) string {
if values := h.getStrings(tag); len(values) > 0 {
return values[0]
}
return ""
}
func (h *header) getUints(tag uint32) []uint64 {
entry := h.entries[tag]
if entry.typ < typeChar || entry.typ > typeInt64 {
return nil
}
values := make([]uint64, 0, entry.count)
for chunk := range slices.Chunk(entry.data, typeSizes[entry.typ]) {
switch len(chunk) {
case 1:
values = append(values, uint64(chunk[0]))
case 2:
values = append(values, uint64(binary.BigEndian.Uint16(chunk)))
case 4:
values = append(values, uint64(binary.BigEndian.Uint32(chunk)))
default:
values = append(values, binary.BigEndian.Uint64(chunk))
}
}
return values
}
func (h *header) getUint(tag uint32) uint64 {
if values := h.getUints(tag); len(values) > 0 {
return values[0]
}
return 0
}
+58 -94
View File
@@ -6,12 +6,11 @@ package rpm
import (
"fmt"
"io"
"strconv"
"strings"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/validation"
"github.com/sassoftware/go-rpmutils"
)
const (
@@ -33,6 +32,11 @@ const (
sIXUSR = 0x40
sIXGRP = 0x8
sIXOTH = 0x1
senseLess = 0x2
senseGreater = 0x4
senseEqual = 0x8
fileGhost = 0x40
)
// https://rpm-software-management.github.io/rpm/manual/spec.html
@@ -100,51 +104,52 @@ type Changelog struct {
// ParsePackage parses the RPM package file
func ParsePackage(r io.Reader) (*Package, error) {
rpm, err := rpmutils.ReadRpm(r)
_, sig, h, err := readHeaders(r)
if err != nil {
return nil, err
}
nevra, err := rpm.Header.GetNEVRA()
if err != nil {
return nil, err
name, ver, rel, arch := h.getString(tagName), h.getString(tagVersion), h.getString(tagRelease), h.getString(tagArch)
if name == "" || ver == "" || rel == "" || arch == "" {
return nil, ErrInvalidPackage
}
epoch := strconv.FormatUint(h.getUint(tagEpoch), 10)
version := fmt.Sprintf("%s-%s", nevra.Version, nevra.Release)
if nevra.Epoch != "" && nevra.Epoch != "0" {
version = fmt.Sprintf("%s-%s", nevra.Epoch, version)
version := fmt.Sprintf("%s-%s", ver, rel)
if epoch != "0" {
version = fmt.Sprintf("%s-%s", epoch, version)
}
p := &Package{
Name: nevra.Name,
Name: name,
Version: version,
VersionMetadata: &VersionMetadata{
Summary: getString(rpm.Header, rpmutils.SUMMARY),
Description: getString(rpm.Header, rpmutils.DESCRIPTION),
License: getString(rpm.Header, rpmutils.LICENSE),
ProjectURL: getString(rpm.Header, rpmutils.URL),
Summary: h.getString(tagSummary),
Description: h.getString(tagDescription),
License: h.getString(tagLicense),
ProjectURL: h.getString(tagURL),
},
FileMetadata: &FileMetadata{
Architecture: nevra.Arch,
Epoch: nevra.Epoch,
Version: nevra.Version,
Release: nevra.Release,
Vendor: getString(rpm.Header, rpmutils.VENDOR),
Group: getString(rpm.Header, rpmutils.GROUP),
Packager: getString(rpm.Header, rpmutils.PACKAGER),
SourceRpm: getString(rpm.Header, rpmutils.SOURCERPM),
BuildHost: getString(rpm.Header, rpmutils.BUILDHOST),
BuildTime: getUInt64(rpm.Header, rpmutils.BUILDTIME),
FileTime: getUInt64(rpm.Header, rpmutils.FILEMTIMES),
InstalledSize: getUInt64(rpm.Header, rpmutils.SIZE),
ArchiveSize: getUInt64(rpm.Header, rpmutils.SIG_PAYLOADSIZE),
Architecture: arch,
Epoch: epoch,
Version: ver,
Release: rel,
Vendor: h.getString(tagVendor),
Group: h.getString(tagGroup),
Packager: h.getString(tagPackager),
SourceRpm: h.getString(tagSourceRpm),
BuildHost: h.getString(tagBuildHost),
BuildTime: h.getUint(tagBuildTime),
FileTime: h.getUint(tagFileMTimes),
InstalledSize: h.getUint(tagSize),
ArchiveSize: sig.getUint(sigTagPayloadSize),
Provides: getEntries(rpm.Header, rpmutils.PROVIDENAME, rpmutils.PROVIDEVERSION, rpmutils.PROVIDEFLAGS),
Requires: getEntries(rpm.Header, rpmutils.REQUIRENAME, rpmutils.REQUIREVERSION, rpmutils.REQUIREFLAGS),
Conflicts: getEntries(rpm.Header, rpmutils.CONFLICTNAME, rpmutils.CONFLICTVERSION, rpmutils.CONFLICTFLAGS),
Obsoletes: getEntries(rpm.Header, rpmutils.OBSOLETENAME, rpmutils.OBSOLETEVERSION, rpmutils.OBSOLETEFLAGS),
Files: getFiles(rpm.Header),
Changelogs: getChangelogs(rpm.Header),
Provides: getEntries(h, tagProvideName, tagProvideVersion, tagProvideFlags),
Requires: getEntries(h, tagRequireName, tagRequireVersion, tagRequireFlags),
Conflicts: getEntries(h, tagConflictName, tagConflictVersion, tagConflictFlags),
Obsoletes: getEntries(h, tagObsoleteName, tagObsoleteVersion, tagObsoleteFlags),
Files: getFiles(h),
Changelogs: getChangelogs(h),
},
}
@@ -155,36 +160,9 @@ func ParsePackage(r io.Reader) (*Package, error) {
return p, nil
}
func getString(h *rpmutils.RpmHeader, tag int) string {
values, err := h.GetStrings(tag)
if err != nil || len(values) < 1 {
return ""
}
return values[0]
}
func getUInt64(h *rpmutils.RpmHeader, tag int) uint64 {
values, err := h.GetUint64s(tag)
if err != nil || len(values) < 1 {
return 0
}
return values[0]
}
func getEntries(h *rpmutils.RpmHeader, namesTag, versionsTag, flagsTag int) []*Entry {
names, err := h.GetStrings(namesTag)
if err != nil || len(names) == 0 {
return nil
}
flags, err := h.GetUint64s(flagsTag)
if err != nil || len(flags) == 0 {
return nil
}
versions, err := h.GetStrings(versionsTag)
if err != nil || len(versions) == 0 {
return nil
}
if len(names) != len(flags) || len(names) != len(versions) {
func getEntries(h *header, namesTag, versionsTag, flagsTag uint32) []*Entry {
names, flags, versions := h.getStrings(namesTag), h.getUints(flagsTag), h.getStrings(versionsTag)
if len(names) == 0 || len(names) != len(flags) || len(names) != len(versions) {
return nil
}
@@ -195,15 +173,15 @@ func getEntries(h *rpmutils.RpmHeader, namesTag, versionsTag, flagsTag int) []*E
}
flags := flags[i]
if (flags&rpmutils.RPMSENSE_GREATER) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 {
if (flags&senseGreater) != 0 && (flags&senseEqual) != 0 {
e.Flags = "GE"
} else if (flags&rpmutils.RPMSENSE_LESS) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 {
} else if (flags&senseLess) != 0 && (flags&senseEqual) != 0 {
e.Flags = "LE"
} else if (flags & rpmutils.RPMSENSE_GREATER) != 0 {
} else if (flags & senseGreater) != 0 {
e.Flags = "GT"
} else if (flags & rpmutils.RPMSENSE_LESS) != 0 {
} else if (flags & senseLess) != 0 {
e.Flags = "LT"
} else if (flags & rpmutils.RPMSENSE_EQUAL) != 0 {
} else if (flags & senseEqual) != 0 {
e.Flags = "EQ"
}
@@ -230,32 +208,29 @@ func getEntries(h *rpmutils.RpmHeader, namesTag, versionsTag, flagsTag int) []*E
return entries
}
func getFiles(h *rpmutils.RpmHeader) []*File {
baseNames, _ := h.GetStrings(rpmutils.BASENAMES)
dirNames, _ := h.GetStrings(rpmutils.DIRNAMES)
dirIndexes, _ := h.GetUint32s(rpmutils.DIRINDEXES)
fileFlags, _ := h.GetUint32s(rpmutils.FILEFLAGS)
fileModes, _ := h.GetUint32s(rpmutils.FILEMODES)
func getFiles(h *header) []*File {
baseNames := h.getStrings(tagBaseNames)
dirNames := h.getStrings(tagDirNames)
dirIndexes := h.getUints(tagDirIndexes)
fileFlags := h.getUints(tagFileFlags)
fileModes := h.getUints(tagFileModes)
files := make([]*File, 0, len(baseNames))
for i := range baseNames {
if len(dirIndexes) <= i {
if i >= len(dirIndexes) || dirIndexes[i] >= uint64(len(dirNames)) {
continue
}
dirIndex := dirIndexes[i]
if len(dirNames) <= int(dirIndex) {
continue
}
var fileType string
var isExecutable bool
if i < len(fileFlags) && (fileFlags[i]&rpmutils.RPMFILE_GHOST) != 0 {
if i < len(fileFlags) && (fileFlags[i]&fileGhost) != 0 {
fileType = "ghost"
} else if i < len(fileModes) {
if (fileModes[i] & sIFMT) == sIFDIR {
fileType = "dir"
} else {
mode := fileModes[i] & ^uint32(sIFMT)
mode := fileModes[i] &^ sIFMT
isExecutable = (mode&sIXUSR) != 0 || (mode&sIXGRP) != 0 || (mode&sIXOTH) != 0
}
}
@@ -270,20 +245,9 @@ func getFiles(h *rpmutils.RpmHeader) []*File {
return files
}
func getChangelogs(h *rpmutils.RpmHeader) []*Changelog {
texts, err := h.GetStrings(rpmutils.CHANGELOGTEXT)
if err != nil || len(texts) == 0 {
return nil
}
authors, err := h.GetStrings(rpmutils.CHANGELOGNAME)
if err != nil || len(authors) == 0 {
return nil
}
times, err := h.GetUint32s(rpmutils.CHANGELOGTIME)
if err != nil || len(times) == 0 {
return nil
}
if len(texts) != len(authors) || len(texts) != len(times) {
func getChangelogs(h *header) []*Changelog {
texts, authors, times := h.getStrings(tagChangelogText), h.getStrings(tagChangelogName), h.getUints(tagChangelogTime)
if len(texts) == 0 || len(texts) != len(authors) || len(texts) != len(times) {
return nil
}
+106 -30
View File
@@ -7,47 +7,61 @@ import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/binary"
"io"
"testing"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/ProtonMail/go-crypto/openpgp/packet"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParsePackage(t *testing.T) {
base64RpmPackageContent := `H4sICFayB2QCAGdpdGVhLXRlc3QtMS4wLjItMS14ODZfNjQucnBtAO2YV4gTQRjHJzl7wbNhhxVF
VNwk2zd2PdvZ9Sxnd3Z3NllNsmF3o6congVFsWFHRWwIImIXfRER0QcRfPBJEXvvBQvWSfZTT0VQ
8TF/MuU33zcz3+zOJGEe73lyuQBRBWKWRzDrEddjuVAkxLMc+lsFUOWfm5bvvReAalWECg/TsivU
dyKa0U61aVnl6wj0Uxe4nc8F92hZiaYE8CO/P0r7/Quegr0c7M/AvoCaGZEIWNGUqMHrhhGROIUT
Zc7gOAOraoQzCNZ0WdU0HpEI5jiB4zlek3gT85wqCBomhomxoGCs8wImWMImbxqKgXVNUKKaqShR
STKVKK9glFUNcf2g+/t27xs16v5x/eyOKftVGlIhyiuvvPLKK6+88sorr7zyyiuvvPKCO5HPnz+v
pGVhhXsTsFVeSstuWR9anwU+Bk3Vch5wTwL3JkHg+8C1gR8A169wj1KdpobAj4HbAT+Be5VewE+h
fz/g52AvBX4N9vHAb4AnA7+F8ePAH8BuA38ELgf+BLzQ50oIeBlw0OdAOXAlP57AGuCsbwGtbgCu
DrwRuAb4bwau6T/PwFbgWsDXgWuD/y3gOmC/B1wI/Bi4AcT3Arih3z9YCNzI9w9m/YKUG4Nd9N9z
pSZgHwrcFPgccFt//OADGE+F/q+Ao+D/FrijzwV1gbv4/QvaAHcFDgF3B5aB+wB3Be7rz1dQCtwP
eDxwMcw3GbgU7AasdwzYE8DjwT4L/CeAvRx4IvBCYA3iWQds+FzpDjABfghsAj8BTgA/A/b8+StX
A84A1wKe5s9fuRB4JpzHZv55rL8a/Dv49vpn/PErR4BvQX8Z+Db4l2W5CH2/f0W5+1fEoeFDBzFp
rE/FMcK4mWQSOzN+aDOIqztW2rPsFKIyqh7sQERR42RVMSKihnzVHlQ8Ag0YLBYNEIajkhmuR5Io
7nlpt2M4nJs0ZNkoYaUyZahMlSfJImr1n1WjFVNCPCaTZgYNGdGL8YN2mX8WHfA/C7ViHJK0pxHG
SrkeTiSI4T+7ubf85yrzRCQRQ5EVxVAjvIBVRY/KRFAVReIkhfARSddNSceayQkGliIKb0q8RAxJ
5QWNVxHIsW3Pz369bw+5jh5y0klE9Znqm0dF57b0HbGy2A5lVUBTZZrqZjdUjYoprFmpsBtHP5d0
+ISltS2yk2mHuC4x+lgJMhgnidvuqy3b0suK0bm+tw3FMxI2zjm7/fA0MtQhplX2s7nYLZ2ZC0yg
CxJZDokhORTJlrlcCvG5OieGBERlVCs7CfuS6WzQ/T2j+9f92BWxTFEcp2IkYccYGp2LYySEfreq
irue4WRF5XkpKovw2wgpq2rZBI8bQZkzxEkiYaNwxnXCCVvHidzIiB3CM2yMYdNWmjDsaLovaE4c
x3a6mLaTxB7rEj3jWN4M2p7uwPaa1GfI8BHFfcZMKhkycnhR7y781/a+A4t7FpWWTupRUtKbegwZ
XMKwJinTSe70uhRcj55qNu3YHtE922Fdz7FTMTq9Q3TbMdiYrrPudMvT44S6u2miu138eC0tTN9D
2CFGHHtQsHHsGCRFDFbXuT9wx6mUTZfseydlkWZeJkW6xOgYjqXT+LA7I6XHaUx2xmUzqelWymA9
rCXI9+D1BHbjsITssqhBNysw0tOWjcpmIh6+aViYPfftw8ZSGfRVPUqKiosZj5R5qGmk/8AjjRbZ
d8b3vvngdPHx3HvMeCarIk7VVSwbgoZVkceEVyOmyUmGxBGNYDVKSFSOGlIkGqWnUZFkiY/wsmhK
Mu0UFYgZ/bYnuvn/vz4wtCz8qMwsHUvP0PX3tbYFUctAPdrY6tiiDtcCddDECahx7SuVNP5dpmb5
9tMDyaXb7OAlk5acuPn57ss9mw6Wym0m1Fq2cej7tUt2LL4/b8enXU2fndk+fvv57ndnt55/cQob
7tpp/pEjDS7cGPZ6BY430+7danDq6f42Nw49b9F7zp6BiKpJb9s5P0AYN2+L159cnrur636rx+v1
7ae1K28QbMMcqI8CqwIrgwg9nTOp8Oj9q81plUY7ZuwXN8Vvs8wbAAA=`
rpmPackageContent, err := base64.StdEncoding.DecodeString(base64RpmPackageContent)
base64RpmsignSignedPackage := `H4sIAAAAAAACA+2ZfWwUxxXA5+y7YIwhJnaaxAFpLSIXE+/5du/242jsAP7ADmA7mBqTGJzZ3bkP
c7497e6Zs0WFCW35UGja4ASXIspHSpOSCDfFSZGaNK0IjRQQSWgaNSgpHw6xKWBDK0pKwJ27fQaH
qlJb5Z9I96Tdmd+8N29mdt7Ond5e2HfxRCai4giGLYJZi5gWy7k9bp7l0P8qDuS6vekHLw86oJqF
UO7ztCyn1wRa76FlLu2UY3PKQRW9JtPKHXadekSOqcBRmzMHaemk1T7g83Z/xwHbX+Yw6A+BfgT0
byM05chGxzwncmSjkm8UOs/UN/RMu3T0hOfxDcdXWD973jhR1+PKaHu/bceUacnnMCf1SNxhHWVP
zB0zGOrPQuvk5r1bBl2vT/+4oiv0flFe9dnX1o9EO8r+9Dsje8Xo1eD8B3cUzuh5IuuTSZs2TNg3
6/h1uel89PWej+LZ2Vkvvevsb1vf9Ooz29e8tZorG/np8Y4X/5zNTt28etd30fcGhXWJ984kxN7i
gtd25l8b+rxiyqSu3O2newq0VbONjS29xU2XB/qWfSi82WRMN3s/OPbb/F7VOPnGC0vXHtlVt9yR
u4378YYHDx9/aOn5Y47Cp89OOuV8C//i0V3fzMppbs+7K/OS65/dcs1zO68P/vyd3pOdR+rnjW75
0cG+F152HR3umlP0wayy00+dH926p3LmZWbNgVn7P3zvpZ0rj8Vu1PzmgHb4bVeisOeTxbVtPuLF
kiL5NV7VNI/ASZxP5DSO07AseziNYEUVZUXhEfFgjvNyPMcrAh/APCd7vQomWgBjr4SxynsxwQIO
8AFN0rCqeCW/EpAkvyAEJD8v4WRMZPu2Lhra/+L+pqahg+of9rT1yXQvc1Fa0pKWtKQlLWlJS1rS
kpa0pCUtXz9J5URGR0e77RzGl/ImVOd6ipYP23kM12qw0eycCmRe7LzIHfTKAB4Cngx8DjhvXN5k
Ir3uBr4AXAx80c6joErgYehfA3wJ9M3AV0DfAvwP4CeAr4L/EPAXoNeBrwN3A98A/r7NyaFSvBk4
A/JCYO90Qt6nBzhpm0mrvcATgbcBZ4P9duBJ9vN07ATOAT4FPBnsB4DvBP0gcC7wBeB8mN9l4Lvt
/hm5wPfY9hn59p457wW9z95n532gbwAuAH4HeKbtP+Mc+JOh/9+B/WB/FXi2zZlTgcvs/plFwOXA
buA5wCJwNXA58Hx7vEzYf2cNMOy/sxbGg/13NoNeg/UuA30EuAX0q8F+OejH9ncF8Fg8KDCf54A1
m51ngQnwX4EDwBeBI8AjwJY9visLOA6cA9xhj+8a278ueB/vt9/HvC1gX2Lr8w7b/l0e4AHoD88z
71OwTyS5At3Kv6JU/hVxaHHDIiaG1ZU4SBgz3t6Ojc4vtWnEVI1wzArrydwo0ia8UkJ8PoUTZUnz
+BQ4GyYvql2CFtT5KhZ4F6PGTtMi7ShkWTFzdmnpzQxnJByNJ1BCFltFH5rxFUv2DKaRWEw8xiyq
X1LJ2JM2mf9bqMOvWNAMxiDtegdhwlHTwpEI0exnt3bAfq4iT3zEo0miJGmyh/diWVL9IvHKkiRw
gkR4j6CqAUHFSoDzaljwSHxA4AWiCTLvVXh57KA3dN2yb/+eb3ebhuo2Yu1JO/pbMDouIm5VZ9I9
YkVfccpbMnOfTLlPTQZU9virVAlHS80Qur2k7iNhZWaF3h4ziGkSrTocIXW4nZjFY7pkS2U4SMe6
1daAOyM6ThmbNbiDNBgkEE7crq41m7tSE/PSBflYDvncotuTKpN3IfVRwefmfG5vKl6zEm9CXDIP
aTS+O29m3MsRy1SEcDRIInqQobMzaby70X9a1fioZzhRknle8Iu+m7+NaNw3CJQ19nlj/GeOEIlE
dFQaN43SiK7iSMozYut5hg0ybCwcIwy7lMYFvRPD0I2ygG60Y4s1iRo3wlYnbY+VsJWt1fWLl9RW
L2ttrP/24oqqMn6sff7C2nkVzc2tcxsbq6hFfV0jwwZIQiWpt9ekQMNOXcnGDN0iqqUbrGkZepSO
HTSIqhsaG1RV1lwVttQQoeZmjKhmmT3fsFJK96HUIFoIW1CwIWxoJEo0VlW5/8IcR6M6XbJt3S76
6M2KR0lZkPowwiqdHzY7o2qIzkmPm2w8uioc1VgLKxFya/JqBJuhsSXQZVGFGhjHSI3RoybRhXg4
aVgYPXX6sMFo/OYformNFbW1jEUSFirwPLKw/571+tmWqjPnDtUeTO1j3AqwMuJkVcai5lWw7OMx
4WVPIMAJmsARhWDZT4hf9GuCx++nb6MkiALv4UVfQBBpJ7+XBPw3Y+Jh+//XF0zyLL8udTU/Rt+h
U9dydmWgQsddtHHGr9eXnHTciVYsR/dO/tip8J/HJ3XvPrSQfPQpW7epddMbZ0Y/+9u+n7zSLBYt
z9m8reHas5v2bBx6cs+NvQUjh3e37D4657PvPLDuj21safm31vX35797+tErT+PQ/crgQP7vh/uK
Tv/q0vSqNfsWJid0X5VuHF3gffzJHdYj5MTaveV94blXts7qKO7OzyhifpmHHM84fkh/MYfXtOa+
OvSXaRmpzyTLHvsXg/M9EcwbAAA=`
rpmPackageContent, err := base64.StdEncoding.DecodeString(base64RpmsignSignedPackage)
assert.NoError(t, err)
zr, err := gzip.NewReader(bytes.NewReader(rpmPackageContent))
assert.NoError(t, err)
decompressed, err := io.ReadAll(zr)
assert.NoError(t, err)
p, err := ParsePackage(zr)
p, err := ParsePackage(bytes.NewReader(decompressed))
assert.NotNil(t, p)
assert.NoError(t, err)
@@ -160,4 +174,66 @@ Mu0UFYgZ/bYnuvn/vz4wtCz8qMwsHUvP0PX3tbYFUctAPdrY6tiiDtcCddDECahx7SuVNP5dpmb5
},
p.FileMetadata.Changelogs,
)
_, sig, h, err := readHeaders(bytes.NewReader(decompressed))
require.NoError(t, err)
headerStart := leadSize + len(sig.raw) + signaturePadding(len(sig.raw))
t.Run("RejectsInvalidHeaders", func(t *testing.T) {
for _, corrupt := range []func([]byte){
func(b []byte) { binary.BigEndian.PutUint32(b[leadSize+12:], maxSignatureData+1) },
func(b []byte) { b[headerStart+len(h.raw)-1] ^= 1 },
func(b []byte) {
binary.BigEndian.PutUint32(b[leadSize+headerIntroSize+3*indexEntrySize+8:], maxHeaderData)
},
func(b []byte) {
copy(b[leadSize+headerIntroSize+4*indexEntrySize+8:][:4], b[leadSize+headerIntroSize+3*indexEntrySize+8:])
},
} {
content := bytes.Clone(decompressed)
corrupt(content)
_, err := ParsePackage(bytes.NewReader(content))
assert.ErrorIs(t, err, ErrInvalidPackage)
}
})
t.Run("RejectsOversizedArrays", func(t *testing.T) {
for _, typ := range []uint32{typeInt8, typeStringArray} {
const count = 1024 * 1024
content := appendUint32s(nil, headerMagic, 0, 1, count)
content = appendUint32s(content, tagFileMTimes, typ, 0, count)
content = append(content, make([]byte, count)...)
_, err := readHeader(bytes.NewReader(content), maxHeaderEntries, maxHeaderData)
assert.ErrorIs(t, err, ErrInvalidPackage)
}
})
t.Run("SignPackage", func(t *testing.T) {
assert.Equal(t, decompressed[leadSize:headerStart], sig.marshalSignature())
entity, err := openpgp.NewEntity("", "", "", &packet.Config{RSABits: 1024})
require.NoError(t, err)
signed, err := SignPackage(bytes.NewReader(decompressed), entity.PrivateKey)
require.NoError(t, err)
content, err := io.ReadAll(signed)
require.NoError(t, err)
_, signedSig, signedHeader, err := readHeaders(bytes.NewReader(content))
require.NoError(t, err)
_, err = openpgp.CheckDetachedSignature(openpgp.EntityList{entity}, bytes.NewReader(signedHeader.raw), bytes.NewReader(signedSig.entries[sigTagRSA].data), nil)
assert.NoError(t, err)
_, err = openpgp.CheckDetachedSignature(openpgp.EntityList{entity}, bytes.NewReader(decompressed[headerStart:]), bytes.NewReader(signedSig.entries[sigTagPGP].data), nil)
assert.NoError(t, err)
assert.Equal(t, decompressed[:leadSize], content[:leadSize])
assert.Equal(t, decompressed[headerStart:], content[leadSize+len(signedSig.raw)+signaturePadding(len(signedSig.raw)):])
signedPackage, err := ParsePackage(bytes.NewReader(content))
require.NoError(t, err)
assert.Equal(t, p, signedPackage)
corruptPayload := bytes.Clone(decompressed)
corruptPayload[len(corruptPayload)-1] ^= 1
_, err = SignPackage(bytes.NewReader(corruptPayload), entity.PrivateKey)
assert.ErrorIs(t, err, ErrInvalidPackage)
})
}
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package rpm
import (
"bytes"
"crypto"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/binary"
"encoding/hex"
"hash"
"io"
"maps"
"slices"
"time"
"github.com/ProtonMail/go-crypto/openpgp/packet"
)
var payloadDigestHashes = map[uint64]func() hash.Hash{1: md5.New, 2: sha1.New, 8: sha256.New, 9: sha512.New384, 10: sha512.New, 11: sha256.New224} // rpm's PGPHASHALGO ids
// SignPackage verifies the payload and replaces all OpenPGP signatures with a header-only and a header+payload one, the latter needed by packages without payload digest
func SignPackage(r io.ReadSeeker, key *packet.PrivateKey) (io.Reader, error) {
lead, sig, h, err := readHeaders(r)
if err != nil {
return nil, err
}
payloadStart, err := r.Seek(0, io.SeekCurrent)
if err != nil {
return nil, err
}
payloadHash, payloadDigest, err := payloadVerifier(sig, h)
if err != nil {
return nil, err
}
headerHash, packageHash := crypto.SHA256.New(), crypto.SHA256.New()
headerHash.Write(h.raw)
packageHash.Write(h.raw)
if _, err := io.Copy(io.MultiWriter(packageHash, payloadHash), r); err != nil {
return nil, err
}
if hex.EncodeToString(payloadHash.Sum(nil)) != payloadDigest {
return nil, ErrInvalidPackage
}
if _, err := r.Seek(payloadStart, io.SeekStart); err != nil {
return nil, err
}
headerSignature, err := signOpenPGP(key, headerHash)
if err != nil {
return nil, err
}
packageSignature, err := signOpenPGP(key, packageHash)
if err != nil {
return nil, err
}
for _, tag := range []uint32{sigTagDSA, sigTagGPG, sigTagReservedSpace} {
delete(sig.entries, tag)
}
sig.entries[sigTagRSA] = headerEntry{typ: typeBin, count: uint32(len(headerSignature)), data: headerSignature}
sig.entries[sigTagPGP] = headerEntry{typ: typeBin, count: uint32(len(packageSignature)), data: packageSignature}
return io.MultiReader(bytes.NewReader(lead), bytes.NewReader(sig.marshalSignature()), bytes.NewReader(h.raw), r), nil
}
func payloadVerifier(sig, h *header) (hash.Hash, string, error) {
if newHash, ok := payloadDigestHashes[h.getUint(tagPayloadDigestAlgo)]; ok {
if digest := h.getString(tagPayloadDigest); digest != "" {
return newHash(), digest, nil
}
}
if md5sum := sig.entries[sigTagMD5]; md5sum.typ == typeBin && len(md5sum.data) > 0 {
hasher := md5.New()
hasher.Write(h.raw)
return hasher, hex.EncodeToString(md5sum.data), nil
}
return nil, "", ErrInvalidPackage
}
func signOpenPGP(key *packet.PrivateKey, digest hash.Hash) ([]byte, error) {
signature := &packet.Signature{
SigType: packet.SigTypeBinary,
CreationTime: time.Now(),
PubKeyAlgo: key.PubKeyAlgo,
Hash: crypto.SHA256,
IssuerKeyId: &key.KeyId,
}
if err := signature.Sign(digest, key, nil); err != nil {
return nil, err
}
var buf bytes.Buffer
err := signature.Serialize(&buf)
return buf.Bytes(), err
}
func (h *header) marshalSignature() []byte {
tags := slices.DeleteFunc(slices.Sorted(maps.Keys(h.entries)), func(tag uint32) bool { return tag <= maxRegionTag })
var index, store []byte
for _, tag := range tags {
entry := h.entries[tag]
if size := typeSizes[entry.typ]; size > 1 {
store = append(store, make([]byte, (size-len(store)%size)%size)...)
}
index = appendUint32s(index, tag, entry.typ, uint32(len(store)), entry.count)
store = append(store, entry.data...)
}
count := uint32(len(tags) + 1)
region := appendUint32s(nil, sigTagRegion, typeBin, uint32(len(store)), indexEntrySize)
store = appendUint32s(store, sigTagRegion, typeBin, -count*indexEntrySize, indexEntrySize)
out := appendUint32s(nil, headerMagic, 0, count, uint32(len(store)))
out = append(append(append(out, region...), index...), store...)
return append(out, make([]byte, signaturePadding(len(store)))...)
}
func appendUint32s(buf []byte, values ...uint32) []byte {
for _, value := range values {
buf = binary.BigEndian.AppendUint32(buf, value)
}
return buf
}
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"regexp"
"sync"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
@@ -176,7 +177,7 @@ func parseMetadataFile(r io.Reader) (*Package, error) {
defer zr.Close()
var spec gemspec
if err := yaml.NewDecoder(zr).Decode(&spec); err != nil {
if err := yaml.NewDecoder(packages.NewLimitedReader(zr, packages.MaxMetadataSize)).Decode(&spec); err != nil {
return nil, err
}
+3 -2
View File
@@ -12,6 +12,7 @@ import (
"strings"
"gitea.dev/modules/json"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"gitea.dev/modules/validation"
@@ -161,7 +162,7 @@ func ParsePackage(sr io.ReaderAt, size int64, mr io.Reader) (*Package, error) {
return nil, ErrManifestFileTooLarge
}
f, err := zr.Open(file.Name)
f, err := file.Open()
if err != nil {
return nil, err
}
@@ -208,7 +209,7 @@ func ParsePackage(sr io.ReaderAt, size int64, mr io.Reader) (*Package, error) {
if mr != nil {
var ssc *SoftwareSourceCode
if err := json.NewDecoder(mr).Decode(&ssc); err != nil {
if err := json.NewDecoder(packages.NewLimitedReader(mr, packages.MaxMetadataSize)).Decode(&ssc); err != nil {
return nil, err
}
p.Metadata.Description = ssc.Description
+1
View File
@@ -99,6 +99,7 @@ func TestParsePackage(t *testing.T) {
nestedManifest := "// swift-tools-version:6.3\n//\n// nested fixture package"
data := writeOrderedZipArchive([][2]string{
{"/Package.swift", nestedManifest},
{"Package.swift", rootManifest},
{"Package@swift-5.5.swift", rootAltManifest},
{"Package@swift-5.7.1.swift", rootPatchAltManifest},
+7 -2
View File
@@ -12,12 +12,17 @@ import (
"gitea.dev/models/db"
packages_model "gitea.dev/models/packages"
"gitea.dev/modules/json"
"gitea.dev/modules/packages"
"gitea.dev/modules/util"
"xorm.io/builder"
)
const LockFile = "terraform.lock"
const (
LockFile = "terraform.lock"
maxLockInfoSize = 64 * 1024
)
// LockInfo is the metadata for a terraform lock.
type LockInfo struct {
@@ -36,7 +41,7 @@ func (l *LockInfo) IsLocked() bool {
func ParseLockInfo(r io.Reader) (*LockInfo, error) {
var lock LockInfo
err := json.NewDecoder(r).Decode(&lock)
err := json.NewDecoder(packages.NewLimitedReader(r, maxLockInfoSize)).Decode(&lock)
if err != nil {
return nil, err
}
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"strings"
"gitea.dev/modules/json"
"gitea.dev/modules/packages"
"gitea.dev/modules/validation"
)
@@ -58,7 +59,7 @@ func ParseMetadataFromBox(r io.Reader) (*Metadata, error) {
// ParseInfoFile parses a info.json file to retrieve the metadata of a Vagrant package
func ParseInfoFile(r io.Reader) (*Metadata, error) {
var values map[string]string
if err := json.NewDecoder(r).Decode(&values); err != nil {
if err := json.NewDecoder(packages.NewLimitedReader(r, packages.MaxMetadataSize)).Decode(&values); err != nil {
return nil, err
}
+3 -1
View File
@@ -39,9 +39,11 @@ type Reader zstd.Decoder
var _ io.ReadCloser = (*Reader)(nil)
const defaultMaxWindowSize = 128 * 1024 * 1024 // the zstd CLI's default, bounds the history buffer an untrusted frame can request
// NewReader returns a new zstd reader.
func NewReader(r io.Reader, opts ...ReaderOption) (*Reader, error) {
zstdR, err := zstd.NewReader(r, opts...)
zstdR, err := zstd.NewReader(r, append([]ReaderOption{zstd.WithDecoderMaxWindow(defaultMaxWindowSize)}, opts...)...)
if err != nil {
return nil, err
}
+9
View File
@@ -11,6 +11,7 @@ import (
"strings"
"testing"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -61,6 +62,14 @@ func TestWriterReader(t *testing.T) {
assert.Equal(t, testData, data)
})
t.Run("rejects window above 128 MiB", func(t *testing.T) {
frameWith256MiBWindow := []byte{0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x90, 0x01, 0x00, 0x00}
reader, err := NewReader(bytes.NewReader(frameWith256MiBWindow))
require.NoError(t, err)
_, err = io.ReadAll(reader)
assert.ErrorIs(t, err, zstd.ErrWindowSizeExceeded)
})
}
func TestSeekableWriterReader(t *testing.T) {
@@ -588,6 +588,8 @@ func PutManifest(ctx *context.Context) {
apiErrorDefined(ctx, namedError)
} else if errors.Is(err, container_model.ErrContainerBlobNotExist) {
apiErrorDefined(ctx, errBlobUnknown)
} else if errors.Is(err, util.ErrInvalidArgument) {
apiErrorDefined(ctx, errManifestInvalid.WithMessage(err.Error()))
} else if errors.Is(err, packages_service.ErrQuotaTotalCount) || errors.Is(err, packages_service.ErrQuotaTypeSize) || errors.Is(err, packages_service.ErrQuotaTotalSize) {
apiError(ctx, http.StatusForbidden, err)
} else {
@@ -137,6 +137,15 @@ func UploadState(ctx *context.Context) {
}
defer buf.Close()
if err := packages_service.CheckSizeQuotaExceeded(ctx, ctx.Doer, ctx.Package.Owner, packages_model.TypeTerraformState, buf.Size()); err != nil {
if errors.Is(err, packages_service.ErrQuotaTypeSize) || errors.Is(err, packages_service.ErrQuotaTotalSize) {
apiError(ctx, http.StatusForbidden, err)
} else {
apiError(ctx, http.StatusInternalServerError, err)
}
return
}
state, err := terraform_module.ParseState(buf)
if err != nil {
log.Error("Error decoding state: %v", err)
+7 -1
View File
@@ -4,6 +4,7 @@
package vagrant
import (
"errors"
"fmt"
"io"
"net/http"
@@ -15,6 +16,7 @@ import (
packages_module "gitea.dev/modules/packages"
vagrant_module "gitea.dev/modules/packages/vagrant"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
"gitea.dev/routers/api/packages/helper"
"gitea.dev/services/context"
packages_service "gitea.dev/services/packages"
@@ -166,7 +168,11 @@ func UploadPackageFile(ctx *context.Context) {
metadata, err := vagrant_module.ParseMetadataFromBox(buf)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
if errors.Is(err, util.ErrInvalidArgument) {
apiError(ctx, http.StatusBadRequest, err)
} else {
apiError(ctx, http.StatusInternalServerError, err)
}
return
}
+3 -16
View File
@@ -4,14 +4,12 @@
package rpm
import (
"bytes"
"io"
"strings"
packages_module "gitea.dev/modules/packages"
rpm_module "gitea.dev/modules/packages/rpm"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/sassoftware/go-rpmutils"
)
func SignPackage(buf *packages_module.HashedBuffer, privateKey string) (*packages_module.HashedBuffer, error) {
@@ -20,20 +18,9 @@ func SignPackage(buf *packages_module.HashedBuffer, privateKey string) (*package
return nil, err
}
h, err := rpmutils.SignRpmStream(buf, keyring[0].PrivateKey, nil)
signed, err := rpm_module.SignPackage(buf, keyring[0].PrivateKey)
if err != nil {
return nil, err
}
signBlob, err := h.DumpSignatureHeader(false)
if err != nil {
return nil, err
}
if _, err := buf.Seek(int64(h.OriginalSignatureHeaderSize()), io.SeekStart); err != nil {
return nil, err
}
// create new buf with signature prefix
return packages_module.CreateHashedBufferFromReader(io.MultiReader(bytes.NewReader(signBlob), buf))
return packages_module.CreateHashedBufferFromReader(signed)
}
+17 -3
View File
@@ -7,6 +7,7 @@ import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/binary"
"encoding/xml"
"fmt"
"io"
@@ -25,7 +26,6 @@ import (
"gitea.dev/tests"
"github.com/ProtonMail/go-crypto/openpgp"
"github.com/sassoftware/go-rpmutils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -868,9 +868,9 @@ gpgkey=%sapi/packages/%s/rpm/repository.key`,
req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s/%s", groupURL, packageName, packageVersion, packageArchitecture, rpmFileName))
resp := MakeRequest(t, req, http.StatusOK)
_, sigs, err := rpmutils.Verify(resp.Body, pub)
header, signature := rpmHeaderAndSignature(resp.Body.Bytes())
_, err = openpgp.CheckDetachedSignature(pub, bytes.NewReader(header), bytes.NewReader(signature), nil)
require.NoError(t, err)
require.NotEmpty(t, sigs)
req = NewRequest(t, "DELETE", fmt.Sprintf("%s/package/%s/%s/%s", groupURL, packageName, packageVersion, packageArchitecture)).
AddBasicAuth(user.Name)
@@ -879,3 +879,17 @@ gpgkey=%sapi/packages/%s/rpm/repository.key`,
})
}
}
func rpmHeaderAndSignature(content []byte) (header, signature []byte) {
const leadSize, introSize, indexEntrySize, sigTagRSA = 96, 16, 16, 268
sigHeader := content[leadSize:]
sigIndex := sigHeader[introSize : introSize+binary.BigEndian.Uint32(sigHeader[8:])*indexEntrySize]
sigStore := sigHeader[introSize+len(sigIndex):]
for entry := range slices.Chunk(sigIndex, indexEntrySize) {
if binary.BigEndian.Uint32(entry) == sigTagRSA {
signature = sigStore[binary.BigEndian.Uint32(entry[8:]):][:binary.BigEndian.Uint32(entry[12:])]
}
}
header = sigStore[(binary.BigEndian.Uint32(sigHeader[12:])+7)/8*8:]
return header[:introSize+binary.BigEndian.Uint32(header[8:])*indexEntrySize+binary.BigEndian.Uint32(header[12:])], signature
}