Support modifying reuploaded tarballs

This commit is contained in:
Eugen Wissner 2023-10-08 12:28:46 +02:00
parent f3beee3e19
commit 5e161c3dad
Signed by: belka
GPG Key ID: A27FDC1E8EE902C0
8 changed files with 115 additions and 56 deletions

View File

@ -85,7 +85,6 @@ end
AUTO_UPDATABLE = { AUTO_UPDATABLE = {
'rdiff-backup' => [SlackBuilder::GitHub.new('rdiff-backup', 'rdiff-backup', 'rdiff-backup')], 'rdiff-backup' => [SlackBuilder::GitHub.new('rdiff-backup', 'rdiff-backup', 'rdiff-backup')],
'librsync' => [SlackBuilder::GitHub.new('librsync', 'librsync')],
'dmd' => [SlackBuilder::LatestText.new('https://downloads.dlang.org/releases/LATEST')] 'dmd' => [SlackBuilder::LatestText.new('https://downloads.dlang.org/releases/LATEST')]
}.freeze }.freeze

View File

@ -15,19 +15,26 @@ import qualified Toml
import qualified Data.ByteString as ByteString import qualified Data.ByteString as ByteString
import Data.Text (Text) import Data.Text (Text)
import qualified Data.Text as Text import qualified Data.Text as Text
import qualified Data.Text.Encoding as Text.Encoding
import qualified Data.Text.IO as Text.IO import qualified Data.Text.IO as Text.IO
import Control.Monad.Trans.Reader (ReaderT(..), asks) import Control.Monad.Trans.Reader (ReaderT(..), asks)
import SlackBuilder.Download import SlackBuilder.Download
import SlackBuilder.Package (Package(..)) import SlackBuilder.Package (Package(..))
import qualified SlackBuilder.Package as Package import qualified SlackBuilder.Package as Package
import Text.URI (mkURI) import Text.URI (mkURI)
import Text.URI.QQ (uri)
import Data.Foldable (for_) import Data.Foldable (for_)
import qualified Text.URI as URI import qualified Text.URI as URI
import System.FilePath ((</>), (<.>)) import System.FilePath ((</>), (<.>), dropExtension, takeBaseName)
import SlackBuilder.Info import SlackBuilder.Info
import Text.Megaparsec (parse, errorBundlePretty) import Text.Megaparsec (parse, errorBundlePretty)
import GHC.Records (HasField(..))
import System.Process
( CmdSpec(..)
, CreateProcess(..)
, StdStream(..)
, callProcess
, withCreateProcess
, waitForProcess
)
autoUpdatable :: [Package] autoUpdatable :: [Package]
autoUpdatable = autoUpdatable =
@ -47,9 +54,7 @@ autoUpdatable =
in Package.Updater latest' template in Package.Updater latest' template
, category = "development" , category = "development"
, name = "universal-ctags" , name = "universal-ctags"
, homepage = [uri|https://ctags.io/|] , reupload = Just []
, requires = pure "%README%"
, reupload = True
} }
, Package , Package
{ latest = { latest =
@ -61,9 +66,7 @@ autoUpdatable =
in Package.Updater latest' template in Package.Updater latest' template
, category = "development" , category = "development"
, name = "composer" , name = "composer"
, homepage = [uri|https://getcomposer.org/|] , reupload = Nothing
, requires = mempty
, reupload = False
} }
, Package , Package
{ latest = { latest =
@ -80,9 +83,7 @@ autoUpdatable =
in Package.Updater latest' template in Package.Updater latest' template
, category = "network" , category = "network"
, name = "jitsi-meet-desktop" , name = "jitsi-meet-desktop"
, homepage = [uri|https://jitsi.org/|] , reupload = Nothing
, requires = mempty
, reupload = False
} }
, Package , Package
{ latest = { latest =
@ -102,9 +103,29 @@ autoUpdatable =
in Package.Updater latest' template in Package.Updater latest' template
, category = "development" , category = "development"
, name = "php82" , name = "php82"
, homepage = [uri|https://www.php.net/|] , reupload = Nothing
, requires = ["postgresql"] }
, reupload = False , Package
{ latest =
let ghArguments = GhArguments
{ owner = "kovidgoyal"
, name = "kitty"
, transform = Nothing
}
latest' = latestGitHub ghArguments $ Text.stripPrefix "v"
templateTail =
[ Package.StaticPlaceholder "/kitty-"
, Package.VersionPlaceholder
, Package.StaticPlaceholder ".tar.xz"
]
template = Package.DownloadTemplate
$ Package.StaticPlaceholder "https://github.com/kovidgoyal/kitty/releases/download/v"
:| Package.VersionPlaceholder
: templateTail
in Package.Updater latest' template
, category = "system"
, name = "kitty"
, reupload = Just [RawCommand "go" ["mod", "vendor"]]
} }
] ]
@ -123,7 +144,11 @@ updatePackageIfRequired package@Package{..} version = do
infoContents <- liftIO $ ByteString.readFile $ repository' </> packagePath infoContents <- liftIO $ ByteString.readFile $ repository' </> packagePath
case parse parseInfoFile packagePath infoContents of case parse parseInfoFile packagePath infoContents of
Right parsedInfoFile -> updatePackage package parsedInfoFile version Right parsedInfoFile
| version == getField @"version" parsedInfoFile ->
liftIO $ Text.IO.putStrLn
$ name <> " is up to date (Version " <> version <> ")."
| otherwise -> updatePackage package parsedInfoFile version
Left errorBundle -> liftIO $ putStr $ errorBundlePretty errorBundle Left errorBundle -> liftIO $ putStr $ errorBundlePretty errorBundle
updatePackage :: Package -> PackageInfo -> Text -> SlackBuilderT () updatePackage :: Package -> PackageInfo -> Text -> SlackBuilderT ()
@ -145,8 +170,6 @@ updatePackage Package{..} info version = do
</> (Text.unpack name <.> "info") </> (Text.unpack name <.> "info")
package' = info package' = info
{ version = version { version = version
, requires = Text.Encoding.encodeUtf8 <$> requires
, homepage = URI.render homepage
, downloads = [download'] , downloads = [download']
, checksums = [checksum] , checksums = [checksum]
} }
@ -156,12 +179,41 @@ updatePackage Package{..} info version = do
commit packagePath version commit packagePath version
where where
handleReupload uri' relativeTarball downloadFileName handleReupload uri' relativeTarball downloadFileName = do
| reupload = repository' <- SlackBuilderT $ asks repository
liftIO (putStrLn $ "Upload the source tarball " <> Text.unpack relativeTarball) case reupload of
Just [] -> uploadTarball relativeTarball downloadFileName
Just commands ->
let tarballPath = repository' </> Text.unpack relativeTarball
packedDirectory = takeBaseName $ dropExtension tarballPath
in liftIO (callProcess "tar" ["xvf", tarballPath])
>> liftIO (traverse (defaultCreateProcess packedDirectory) commands)
>> liftIO (callProcess "tar" ["Jcvf", tarballPath, packedDirectory])
>> uploadTarball relativeTarball downloadFileName
Nothing -> pure uri'
uploadTarball relativeTarball downloadFileName
= liftIO (putStrLn $ "Upload the source tarball " <> Text.unpack relativeTarball)
>> uploadCommand relativeTarball ("/" <> name) >> uploadCommand relativeTarball ("/" <> name)
>> liftIO (mkURI $ "https://download.dlackware.com/hosted-sources/" <> name <> "/" <> downloadFileName) >> liftIO (mkURI $ "https://download.dlackware.com/hosted-sources/" <> name <> "/" <> downloadFileName)
| otherwise = pure uri' defaultCreateProcess cwd' cmdSpec
= flip withCreateProcess (const . const . const waitForProcess)
$ CreateProcess
{ use_process_jobs = False
, std_out = Inherit
, std_in = NoStream
, std_err = Inherit
, new_session = False
, env = Nothing
, detach_console = False
, delegate_ctlc = False
, cwd = Just cwd'
, create_new_console = False
, create_group = False
, cmdspec = cmdSpec
, close_fds = True
, child_user = Nothing
, child_group = Nothing
}
main :: IO () main :: IO ()
main = do main = do

View File

@ -5,6 +5,4 @@ download_url = "https://example.com/some/path"
remote_path = "example.com:/srv/httpd/some/path" remote_path = "example.com:/srv/httpd/some/path"
[maintainer] [maintainer]
name = "Maintainer Name"
email = "maintainer@example.com"
signature = false signature = false

View File

@ -17,10 +17,8 @@ data Settings = Settings
, maintainer :: MaintainerSettings , maintainer :: MaintainerSettings
} deriving (Eq, Show) } deriving (Eq, Show)
data MaintainerSettings = MaintainerSettings newtype MaintainerSettings = MaintainerSettings
{ name :: !Text { signature :: Bool
, email :: !Text
, signature :: !Bool
} deriving (Eq, Show) } deriving (Eq, Show)
settingsCodec :: Toml.TomlCodec Settings settingsCodec :: Toml.TomlCodec Settings
@ -34,6 +32,4 @@ settingsCodec = Settings
maintainerSettingsCodec :: Toml.TomlCodec MaintainerSettings maintainerSettingsCodec :: Toml.TomlCodec MaintainerSettings
maintainerSettingsCodec = MaintainerSettings maintainerSettingsCodec = MaintainerSettings
<$> Toml.text "name" .= name <$> Toml.bool "signature" .= signature
<*> Toml.text "email" .= email
<*> Toml.bool "signature" .= signature

View File

@ -174,8 +174,8 @@ generate pkg = Lazy.Text.toStrict $ Text.Builder.toLazyText builder
<> "HOMEPAGE=\"" <> Text.Builder.fromText (homepage pkg) <> "\"\n" <> "HOMEPAGE=\"" <> Text.Builder.fromText (homepage pkg) <> "\"\n"
<> generateMultiEntry "DOWNLOAD" (render <$> downloads pkg) <> generateMultiEntry "DOWNLOAD" (render <$> downloads pkg)
<> generateMultiEntry "MD5SUM" (digestToText <$> checksums pkg) <> generateMultiEntry "MD5SUM" (digestToText <$> checksums pkg)
<> generateMultiEntry "DOWNLOAD_x86_64" (render <$> downloads pkg) <> generateMultiEntry "DOWNLOAD_x86_64" (render <$> downloadX64 pkg)
<> generateMultiEntry "MD5SUM_x86_64" (digestToText <$> checksums pkg) <> generateMultiEntry "MD5SUM_x86_64" (digestToText <$> checksumX64 pkg)
<> "REQUIRES=\"" <> fromByteStringWords (requires pkg) <> "\"\n" <> "REQUIRES=\"" <> fromByteStringWords (requires pkg) <> "\"\n"
<> "MAINTAINER=\"" <> Text.Builder.fromText (maintainer pkg) <> "\"\n" <> "MAINTAINER=\"" <> Text.Builder.fromText (maintainer pkg) <> "\"\n"
<> "EMAIL=\"" <> Text.Builder.fromText (email pkg) <> "\"\n" <> "EMAIL=\"" <> Text.Builder.fromText (email pkg) <> "\"\n"

View File

@ -17,15 +17,14 @@ import qualified Text.URI as URI
import Crypto.Hash (Digest, MD5) import Crypto.Hash (Digest, MD5)
import SlackBuilder.Trans import SlackBuilder.Trans
import Control.Monad.Catch (MonadThrow) import Control.Monad.Catch (MonadThrow)
import System.Process (CmdSpec(..))
-- | Contains information how a package can be updated. -- | Contains information how a package can be updated.
data Package = Package data Package = Package
{ latest :: Updater { latest :: Updater
, category :: Text , category :: Text
, name :: Text , name :: Text
, homepage :: URI , reupload :: Maybe [CmdSpec]
, requires :: [Text]
, reupload :: Bool
} }
-- | Download URI with the MD5 checksum of the target. -- | Download URI with the MD5 checksum of the target.

View File

@ -25,6 +25,7 @@ common dependencies
modern-uri ^>= 0.3.6, modern-uri ^>= 0.3.6,
memory ^>= 0.18, memory ^>= 0.18,
parser-combinators ^>= 1.3, parser-combinators ^>= 1.3,
process ^>= 1.6.18,
text ^>= 2.0, text ^>= 2.0,
tomland ^>= 1.3.3, tomland ^>= 1.3.3,
transformers ^>= 0.5.6, transformers ^>= 0.5.6,
@ -69,7 +70,6 @@ executable slackbuilder
conduit ^>= 1.3.5, conduit ^>= 1.3.5,
http-client ^>= 0.7, http-client ^>= 0.7,
optparse-applicative ^>= 0.18.1, optparse-applicative ^>= 0.18.1,
process ^>= 1.6.17,
req ^>= 3.13, req ^>= 3.13,
slackbuilder, slackbuilder,
unordered-containers ^>= 0.2.19, unordered-containers ^>= 0.2.19,

View File

@ -21,18 +21,28 @@ parseInfoFile'
parseInfoFile' = parse parseInfoFile "" parseInfoFile' = parse parseInfoFile ""
infoDownload0 :: ByteString infoDownload0 :: ByteString
infoDownload0 = "PKGNAM=\"pkgnam\"\n\ infoDownload0 = "PRGNAM=\"pkgnam\"\n\
\VERSION=\"1.2.3\"\n\ \VERSION=\"1.2.3\"\n\
\HOMEPAGE=\"homepage\"\n\ \HOMEPAGE=\"homepage\"\n\
\DOWNLOAD=\"\"\n\ \DOWNLOAD=\"\"\n\
\MD5SUM=\"\"\n" \MD5SUM=\"\"\n\
\DOWNLOAD_x86_64=\"\"\n\
\MD5SUM_x86_64=\"\"\n\
\REQUIRES=\"\"\n\
\MAINTAINER=\"Z\"\n\
\EMAIL=\"test@example.com\"\n"
infoDownload1 :: ByteString infoDownload1 :: ByteString
infoDownload1 = "PKGNAM=\"pkgnam\"\n\ infoDownload1 = "PRGNAM=\"pkgnam\"\n\
\VERSION=\"1.2.3\"\n\ \VERSION=\"1.2.3\"\n\
\HOMEPAGE=\"homepage\"\n\ \HOMEPAGE=\"homepage\"\n\
\DOWNLOAD=\"https://dlackware.com/download.tar.gz\"\n\ \DOWNLOAD=\"https://dlackware.com/download.tar.gz\"\n\
\MD5SUM=\"0102030405060708090a0b0c0d0e0f10\"\n" \MD5SUM=\"0102030405060708090a0b0c0d0e0f10\"\n\
\DOWNLOAD_x86_64=\"\"\n\
\MD5SUM_x86_64=\"\"\n\
\REQUIRES=\"\"\n\
\MAINTAINER=\"Z\"\n\
\EMAIL=\"test@example.com\"\n"
maybeToDoubleList :: forall a. Maybe a -> [a] maybeToDoubleList :: forall a. Maybe a -> [a]
maybeToDoubleList xs = [y | x <- maybeToList xs, y <- [x, x]] maybeToDoubleList xs = [y | x <- maybeToList xs, y <- [x, x]]
@ -60,7 +70,7 @@ spec = do
describe "generate" $ do describe "generate" $ do
it "generates an .info file without downloads" $ it "generates an .info file without downloads" $
let given = PackageInfo "pkgnam" "1.2.3" "homepage" [] [] let given = PackageInfo "pkgnam" "1.2.3" "homepage" [] [] [] [] [] "Z" "test@example.com"
in generate given `shouldBe` Text.decodeUtf8 infoDownload0 in generate given `shouldBe` Text.decodeUtf8 infoDownload0
it "splits multiple downloads into multiple lines" $ it "splits multiple downloads into multiple lines" $
@ -69,21 +79,26 @@ spec = do
checksums' = maybeToDoubleList checksums' = maybeToDoubleList
$ digestFromByteString (ByteString.pack [1.. 16]) $ digestFromByteString (ByteString.pack [1.. 16])
given = PackageInfo given = PackageInfo
"pkgnam" "1.2.3" "homepage" downloads' checksums' "pkgnam" "1.2.3" "homepage" downloads' checksums' [] [] [] "Z" "test@example.com"
expected = "PKGNAM=\"pkgnam\"\n\ expected = "PRGNAM=\"pkgnam\"\n\
\VERSION=\"1.2.3\"\n\ \VERSION=\"1.2.3\"\n\
\HOMEPAGE=\"homepage\"\n\ \HOMEPAGE=\"homepage\"\n\
\DOWNLOAD=\"https://dlackware.com/download.tar.gz \\\n\ \DOWNLOAD=\"https://dlackware.com/download.tar.gz \\\n\
\ https://dlackware.com/download.tar.gz\"\n\ \ https://dlackware.com/download.tar.gz\"\n\
\MD5SUM=\"0102030405060708090a0b0c0d0e0f10 \\\n\ \MD5SUM=\"0102030405060708090a0b0c0d0e0f10 \\\n\
\ 0102030405060708090a0b0c0d0e0f10\"\n" \ 0102030405060708090a0b0c0d0e0f10\"\n\
\DOWNLOAD_x86_64=\"\"\n\
\MD5SUM_x86_64=\"\"\n\
\REQUIRES=\"\"\n\
\MAINTAINER=\"Z\"\n\
\EMAIL=\"test@example.com\"\n"
in generate given `shouldBe` expected in generate given `shouldBe` expected
it "prints the checksum as a sequence of hexadecimal numbers" $ it "prints the checksum as a sequence of hexadecimal numbers" $
let downloads' = maybeToList let downloads' = maybeToList
$ mkURI "https://dlackware.com/download.tar.gz" $ mkURI "https://dlackware.com/download.tar.gz"
given = PackageInfo given = PackageInfo
"pkgnam" "1.2.3" "homepage" downloads' checksumSample "pkgnam" "1.2.3" "homepage" downloads' checksumSample [] [] [] "Z" "test@example.com"
in generate given `shouldBe` Text.decodeUtf8 infoDownload1 in generate given `shouldBe` Text.decodeUtf8 infoDownload1
describe "updateDownloadVersion" $ do describe "updateDownloadVersion" $ do
@ -91,7 +106,7 @@ spec = do
let downloads' = maybeToList let downloads' = maybeToList
$ mkURI "https://dlackware.com/download-1.2.3.tar.gz" $ mkURI "https://dlackware.com/download-1.2.3.tar.gz"
testPackage = PackageInfo testPackage = PackageInfo
"pkgnam" "1.2.3" "homepage" downloads' checksumSample "pkgnam" "1.2.3" "homepage" downloads' checksumSample [] [] [] "Z" "test@example.com"
expected = maybeToList expected = maybeToList
$ mkURI "https://dlackware.com/download-2.3.4.tar.gz" $ mkURI "https://dlackware.com/download-2.3.4.tar.gz"
actual = updateDownloadVersion testPackage "2.3.4" Nothing actual = updateDownloadVersion testPackage "2.3.4" Nothing
@ -101,7 +116,7 @@ spec = do
let downloads' = maybeToList let downloads' = maybeToList
$ mkURI "https://dlackware.com/1.2/download.tar.gz" $ mkURI "https://dlackware.com/1.2/download.tar.gz"
testPackage = PackageInfo testPackage = PackageInfo
"pkgnam" "1.2.3" "homepage" downloads' checksumSample "pkgnam" "1.2.3" "homepage" downloads' checksumSample [] [] [] "Z" "test@example.com"
expected = maybeToList expected = maybeToList
$ mkURI "https://dlackware.com/2.3/download.tar.gz" $ mkURI "https://dlackware.com/2.3/download.tar.gz"
actual = updateDownloadVersion testPackage "2.3.4" Nothing actual = updateDownloadVersion testPackage "2.3.4" Nothing
@ -110,8 +125,8 @@ spec = do
it "updates gnome version" $ it "updates gnome version" $
let downloads' = maybeToList let downloads' = maybeToList
$ mkURI "https://download.gnome.org/core/3.36/3.36.0/sources/gnome-calendar-3.36.0.tar.xz" $ mkURI "https://download.gnome.org/core/3.36/3.36.0/sources/gnome-calendar-3.36.0.tar.xz"
testPackage = PackageInfo testPackage = PackageInfo "gnome-calendar" "3.36.0" "https://wiki.gnome.org/Core/Calendar"
"gnome-calendar" "3.36.0" "https://wiki.gnome.org/Core/Calendar" downloads' checksumSample downloads' checksumSample [] [] [] "Z" "test@example.com"
expected = maybeToList expected = maybeToList
$ mkURI "https://download.gnome.org/core/3.36/3.36.4/sources/gnome-calendar-3.36.2.tar.xz" $ mkURI "https://download.gnome.org/core/3.36/3.36.4/sources/gnome-calendar-3.36.2.tar.xz"
actual = updateDownloadVersion testPackage "3.36.2" $ Just "3.36.4" actual = updateDownloadVersion testPackage "3.36.2" $ Just "3.36.4"
@ -121,7 +136,7 @@ spec = do
let downloads' = maybeToList let downloads' = maybeToList
$ mkURI "https://dlackware.com/gnome-contacts-3.36.tar.xz" $ mkURI "https://dlackware.com/gnome-contacts-3.36.tar.xz"
testPackage = PackageInfo testPackage = PackageInfo
"gnome-contacts" "3.36" "homepage" downloads' checksumSample "gnome-contacts" "3.36" "homepage" downloads' checksumSample [] [] [] "Z" "test@example.com"
expected = maybeToList expected = maybeToList
$ mkURI "https://dlackware.com/gnome-contacts-3.36.2.tar.xz" $ mkURI "https://dlackware.com/gnome-contacts-3.36.2.tar.xz"
actual = updateDownloadVersion testPackage "3.36.2" Nothing actual = updateDownloadVersion testPackage "3.36.2" Nothing
@ -132,8 +147,8 @@ spec = do
let downloads' = maybeToList let downloads' = maybeToList
$ mkURI "https://dlackware.com/1.2/download.tar.gz" $ mkURI "https://dlackware.com/1.2/download.tar.gz"
testPackage = PackageInfo testPackage = PackageInfo
"pkgnam" "1.2.3" "homepage" downloads' checksumSample "pkgnam" "1.2.3" "homepage" downloads' checksumSample [] [] [] "Z" "test@example.com"
expected = PackageInfo expected = PackageInfo
"pkgnam" "2.3.4" "homepage" downloads' checksumSample "pkgnam" "2.3.4" "homepage" downloads' checksumSample [] [] [] "Z" "test@example.com"
given = update testPackage "2.3.4" downloads' checksumSample given = update testPackage "2.3.4" downloads' checksumSample
in given `shouldBe` expected in given `shouldBe` expected