LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

depends: sanity-check sources and cached builds

Browse source 
In some cases (Travis), sources and build caches may be moved around in-between
builds, and we can't necessarily trust that everything is still intact.

This introduces pre-build checks that verify against stashed checksums.

Note that this will cause all sources to be re-downloaded, since cached sources
weren't trustworthy before this.
This commit is contained in:
Cory Fields 2015-05-13 21:12:04 -04:00
parent 351f73ecd5
commit 235b3a789d
4 changed files with 49 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
depends/Makefile
View file

@ -91,12 +91,12 @@ include funcs.mk
toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages)
$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
$(AT)rm -rf $(@D)
$(AT)mkdir -p $(@D)
$(AT)echo copying packages: $|
$(AT)echo copying packages: $^
$(AT)echo to: $(@D)
$(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); )
$(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); )
$(AT)touch $@
$(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
@ -121,8 +121,35 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
$< > $@
$(AT)touch $@
install: $(host_prefix)/share/config.site
download-one: $(all_sources)
define check_or_remove_cached
mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \
$(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \
( rm -f $($(package)_cached_checksum); \
if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi )
endef
define check_or_remove_sources
mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \
$(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \
( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \
rm -f $($(package)_all_sources) $($(1)_fetched))
endef
check-packages:
@$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));)
check-sources:
@$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));)
$(host_prefix)/share/config.site: check-packages
check-packages: check-sources
install: check-packages $(host_prefix)/share/config.site
download-one: check-sources $(all_sources)
download-osx:
@$(MAKE) -s HOST=x86_64-apple-darwin11 download-one
download-linux:
@ -130,4 +157,5 @@ download-linux:
download-win:
@$(MAKE) -s HOST=x86_64-w64-mingw32 download-one
download: download-osx download-linux download-win
.PHONY: install cached download-one download-osx download-linux download-win download
.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources

4
depends/README.packages
View file

@ -30,7 +30,9 @@ These variables are optional:
Names of any other packages that this one depends on.
$(package)_patches:
Filenames of any patches needed to build the package
$(package)_extra_sources:
Any extra files that will be fetched via $(package)_fetch_cmds. These are
specified so that they can be fetched and verified via 'make download'.
Build Variables:
After defining the main identifiers, build variables may be added or customized

13
depends/funcs.mk
View file

@ -53,12 +53,14 @@ $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix)
$(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)
$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version)
$(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir)
$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash
$(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id)
$(1)_prefixbin:=$($($(1)_type)_prefix)/bin/
$(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz
$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources)
#stamps
$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name)
$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash
$(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted
$(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed
$(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned
@ -154,7 +156,10 @@ endef
define int_add_cmds
$($(1)_fetched):
$(AT)mkdir -p $$(@D) $(SOURCES_PATH)
$(AT)rm -f $$@
$(AT)touch $$@
$(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1))
$(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);)
$(AT)touch $$@
$($(1)_extracted): | $($(1)_fetched)
$(AT)echo Extracting $(1)...
@ -195,10 +200,12 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed)
$(AT)rm -rf $$(@D) && mkdir -p $$(@D)
$(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@)
$(AT)rm -rf $($(1)_staging_dir)
$($(1)_cached_checksum): $($(1)_cached)
$(AT)cd $$(@D); $(build_SHA256SUM) $$(<F) > $$(@)
.PHONY: $(1)
$(1): | $($(1)_cached)
.SECONDARY: $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched)
$(1): | $($(1)_cached_checksum)
.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched)
endef

2
depends/packages/native_cctools.mk
View file

@ -9,6 +9,8 @@ $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_versi
$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz
$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz
$(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7
$(package)_extra_sources=$($(package)_clang_file_name)
define $(package)_fetch_cmds
$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash))