Storable-3.12_02, Makefile integration

Enable >2GB AvFILL check on store_hook (64bit).

Too many references returned by STORABLE_freeze.
It wrapped around previously. Fixes Coverity CID #187854.
This is a security issue on 64bit cperl and perl5, as both allow
SSize_t arrays.

Move __Storable__.pm into Storable_pm.PL
[cperl #374]

Merge Makefile recipes from blead.

Author: rurban

.gitignore

@@ -135,6 +135,7 @@ lib/unicore/lib/
 lib/unicore/mktables.lst
 lib/B/Stash.pm
 lib/ByteLoader.pm
+lib/Storable/
 
 # generated by WinCE build
 xlib/

MANIFEST

@@ -3639,9 +3639,8 @@ dist/Storable/hints/linux.pl		Hint for Storable for named architecture
 dist/Storable/Makefile.PL		Storable extension
 dist/Storable/README			Storable extension
 dist/Storable/stacksize			compute stack sizes
-dist/Storable/stacksize_in.h		computed stack sizes
-dist/Storable/Storable.pm		Storable extension
 dist/Storable/Storable.xs		Storable extension
+dist/Storable/Storable_pm.PL		Storable extension
 dist/Storable/t/attach.t		Check STORABLE_attach doesn't create objects unnecessarily
 dist/Storable/t/attach_errors.t		Trigger and test STORABLE_attach errors
 dist/Storable/t/attach_singleton.t	Test STORABLE_attach for the Singleton pattern
@@ -3665,6 +3664,7 @@ dist/Storable/t/HAS_ATTACH.pm		For auto-requiring of modules for STORABLE_attach
 dist/Storable/t/HAS_HOOK.pm		For auto-requiring of modules for STORABLE_thaw
 dist/Storable/t/HAS_OVERLOAD.pm		For auto-requiring of mdoules for overload
 dist/Storable/t/huge.t			See how Storable handles huge data
+dist/Storable/t/hugeids.t
 dist/Storable/t/integer.t		See if Storable works
 dist/Storable/t/interwork56.t		Test compatibility kludge for 64bit data under 5.6.x
 dist/Storable/t/just_plain_nasty.t	See if Storable works
@@ -3676,6 +3676,7 @@ dist/Storable/t/make_overload.pl	Make test data for overload.t
 dist/Storable/t/malice.t		See if Storable copes with corrupt files
 dist/Storable/t/overload.t		See if Storable works
 dist/Storable/t/recurse.t		See if Storable works
+dist/Storable/t/regexp.t
 dist/Storable/t/restrict.t		See if Storable works
 dist/Storable/t/retrieve.t		See if Storable works
 dist/Storable/t/robust.t		See if it survives mangled %INC

Makefile.SH

@@ -177,11 +177,15 @@ esac
 : is Cwd static or dynamic
 static_cwd='define'
 cwd_dep='$(PERL_EXE)'
+storable_limit_dep=''
+storable_type=''
 list_util_dep='$(PERL_EXE)'
 for f in $dynamic_ext; do
    case $f in
        Cwd) static_cwd='undef'; cwd_dep=lib/auto/Cwd/Cwd.$dlext ;;
-       List/Util) list_util_dep=lib/auto/List/Util/Util.$dlext
+       List/Util) list_util_dep=lib/auto/List/Util/Util.$dlext ;;
+       Storable) storable_limit_dep=lib/auto/Storable/Storable.$dlext
+                 storable_type='dynamic' ;;
    esac
 done
 
@@ -222,6 +226,7 @@ $this_target: lib/auto/re/re\$(LIB_EXT)" ;;
 $this_target: lib/auto/List/Util/Util\$(LIB_EXT)" ;;
 	Unicode/Normalize) extra_dep="$extra_dep
 $this_target: uni.data" ;;
+        Storable) storable_type='static' ;;
     esac
 done
 
@@ -640,7 +645,7 @@ lintflags = \
 .c.s:	perl.h config.h
 	`$(CCCMDSRC)` -S $*.c
 
-all: $(FIRSTMAKEFILE) $(MINIPERL_EXE) $(generated_pods) $(private) $(unidatafiles) $(public) $(dynamic_ext) $(nonxs_ext) extras.make $(MANIFEST_SRT)
+all: $(FIRSTMAKEFILE) $(MINIPERL_EXE) $(generated_pods) $(private) $(unidatafiles) $(public) $(dynamic_ext) $(nonxs_ext) extras.make $(MANIFEST_SRT) common_build
 	@echo " ";
 	@echo "	Everything is up to date. Type '$(MAKE) test' to run test suite."
 
@@ -1179,7 +1184,7 @@ esac
 
 $spitshell >>$Makefile <<'!NO!SUBS!'
 
-.PHONY: preplibrary rebuild_storable
+.PHONY: preplibrary common_build
 preplibrary: $(MINIPERL_EXE) $(CONFIGPM) $(PREPLIBRARY_LIBPERL)
 
 $(CONFIGPM_FROM_CONFIG_SH): $(CONFIGPOD)
@@ -1197,16 +1202,6 @@ uni.data: $(MINIPERL_EXE) $(CONFIGPM) lib/unicore/mktables $(nonxs_ext)
 #	can in this makefile to decide if needs to run or not
 #	touch uni.data
 
-!NO!SUBS!
-$spitshell >>$Makefile <<!GROK!THIS!
-
-rebuild_storable: \$(PERL_EXE) $cwd_dep
-	-\$(RUN_PERL) dist/Storable/stacksize --core
-	\$(MAKE) -C dist/Storable PERL_CORE=1
-
-!GROK!THIS!
-$spitshell >>$Makefile <<'!NO!SUBS!'
-
 # $(PERL_EXE) and ext because pod_lib.pl needs Digest::MD5
 # But also this ensures that all extensions are built before we try to scan
 # them, which picks up Devel::PPPort's documentation.
@@ -1238,7 +1233,7 @@ extra.pods: $(MINIPERL_EXE)
 	    echo "pod/perl"$$nx".pod" >> extra.pods ; \
 	done
 
-extras.make: $(PERL_EXE) rebuild_storable
+extras.make: $(PERL_EXE)
 	-@test ! -s extras.lst || PATH="`pwd`:${PATH}" PERL5LIB="`pwd`/lib" $(RUN_PERL) -MCPAN -e '@ARGV&&make(@ARGV)' `cat extras.lst`
 
 extras.test: $(PERL_EXE)
@@ -1258,6 +1253,25 @@ no_install no-install: install-notify
 install: install-all
 !NO!SUBS!
 
+if test "$storable_type" != "" ; then
+
+case "$static_cwd" in
+undef) storable_limit_dep="$storable_limit_dep lib/auto/Cwd/Cwd.$dlext" ;;
+esac
+
+$spitshell >>$Makefile <<EOT
+
+dist/Storable/lib/Storable/Limit.pm : \$(PERL_EXE) dist/Storable/stacksize $storable_limit_dep
+	cd dist/Storable ; \$(LDLIBPTH) \$(MAKE) lib/Storable/Limit.pm
+
+lib/Storable/Limit.pm : dist/Storable/lib/Storable/Limit.pm
+	test -d lib/Storable || mkdir lib/Storable
+	cp dist/Storable/lib/Storable/Limit.pm lib/Storable/Limit.pm
+EOT
+
+common_build_deps="$common_build_deps lib/Storable/Limit.pm"
+
+fi
 
 for name in all notify silent strip verbose; do
     flags="--$name";
@@ -1640,14 +1654,14 @@ test_prep_pre: preplibrary utilities $(nonxs_ext)
 case "$targethost" in
 '') $spitshell >>$Makefile <<'!NO!SUBS!'
 test_prep test-prep: test_prep_pre $(MINIPERL_EXE) $(unidatafiles) $(PERL_EXE) \
-	$(dynamic_ext) extras.make $(TEST_PERL_DLL) runtests $(generated_pods)
+	$(dynamic_ext) extras.make $(TEST_PERL_DLL) runtests $(generated_pods) common_build
 	cd t && (rm -f $(PERL_EXE); $(LNS) ../$(PERL_EXE) $(PERL_EXE))
 !NO!SUBS!
     ;;
 *) $spitshell >>$Makefile <<!GROK!THIS!
 test_prep test-prep: test_prep_pre \$(MINIPERL_EXE) \$(unidatafiles) \$(PERL_EXE) \
 	\$(dynamic_ext) extras.make \$(TEST_PERL_DLL) runtests \
-	\$(generated_pods)
+	\$(generated_pods) common_build
 	$to libperl.*
 	$to t/*
 	$to lib/*
@@ -1763,9 +1777,9 @@ esac
 $spitshell >>$Makefile <<'!NO!SUBS!'
 
 # minitest can't depend on lib/Config.pm because that might be where miniperl
-# is crashing.
+# is crashing. ditto for mktables
 minitest_prep:
-	-@test -f lib/Config.pm || $(MAKE) lib/Config.pm $(unidatafiles)
+	-@test -f lib/Config.pm -a -f lib/unicore/Name.pm || $(MAKE) lib/Config.pm $(unidatafiles)
 	@echo " "
 	@echo "You may see some irrelevant test failures if you have been unable"
 	@echo "to build lib/Config.pm, or the Unicode data files."
@@ -1797,6 +1811,17 @@ test_reonly test-reonly: test_prep_reonly
 test_porting test-porting: test_prep
 	cd t && $(RUN_PERL) harness porting/*.t ../lib/diagnostics.t
 
+!NO!SUBS!
+
+$spitshell>>$Makefile <<!GROK!THIS!
+
+# intended as a common way to add dependencies to test-prep and all
+common_build: $common_build_deps
+
+!GROK!THIS!
+
+$spitshell >>$Makefile <<'!NO!SUBS!'
+
 # Handy way to run cperlbug -ok without having to install and run the
 # installed cperlbug. We don't re-run the tests here - we trust the user.
 # Please *don't* use this unless all tests pass.
@@ -1842,11 +1867,17 @@ distcheck: FORCE
 
 .PHONY: ctags
 
-TAGS: $(c1) $(c2) $(c3) $(c4) $(c5) $(h)
-	etags $(c1) $(c2) $(c3) $(c4) $(c5) $(h)
+TAGS: $(c1) $(c2) $(c3) $(c4) $(c5) $(h) $(inc)
+	etags $(c1) $(c2) $(c3) $(c4) $(c5) $(h) $(inc) mg_*.h
+
+FULLTAGS: $(c1) $(c2) $(c3) $(c4) $(c5) $(h) $(inc)
+	etags $(c1) $(c2) $(c3) $(c4) $(c5) $(h) $(inc) mg_*.h \
+	      `find cpan dist ext -type f -name \*.[ch]`
 
-FULLTAGS: $(c1) $(c2) $(c3) $(c4) $(c5) $(h)
-	etags $(c1) $(c2) $(c3) $(c4) $(c5) $(h) `find cpan dist ext -type f -name \*.[ch]`
+GTAGS: $(c1) $(c2) $(c3) $(c4) $(c5) $(h) $(inc)
+	echo $(c1) $(c2) $(c3) $(c4) $(c5) $(h) $(inc) mg_*.h | \
+		tr ' ' $(TRNL) > gtags.files
+	gtags -v --file gtags.files
 
 ctags:
 	ctags -f Tags -N --totals --languages=c --langmap=c:+.h --exclude=opmini.c --exclude=perlmini.c *.c *.h

Porting/Maintainers.pl

@@ -1283,8 +1283,11 @@ package Maintainers;
     },
 
     'Storable' => {
-        'DISTRIBUTION' => 'RURBAN/Storable-3.05_14.tar.gz',
+        'DISTRIBUTION' => 'RURBAN/Storable-3.12_02.tar.gz',
         'FILES'        => q[dist/Storable],
+        'EXCLUDED'     => [
+            qw( ptr_table.h t/leaks_refcnt.t .travis.yml appveyor.yml ),
+        ]
     },
 
     'Sys::Syslog' => {

dist/.gitignore

@@ -7,6 +7,7 @@ Makefile
 Makefile.PL
 ppport.h
 Storable/storable-testfile.*
-Storable/stacksize.h
+Storable/Storable.pm
+Storable/lib/Storable/Limit.pm
 Time-HiRes/*.inc
 Time-HiRes/xdefine

dist/Module-CoreList/lib/Module/CoreList.pm

@@ -17779,6 +17779,7 @@ our %delta :const = (
             'Scalar::Util'          => '1.50_11',
             'Sub::Util'             => '1.50_11',
             'Socket'                => '2.027_04',
+            'Storable'              => '3.12_02',
         },
         removed => {
         }

dist/Storable/ChangeLog

@@ -1,3 +1,84 @@
+2018-04-27 20:40:00 xsawyerx
+    version 3.11
+        * Fix Strawberry Perl build failures.
+
+2018-04-21 22:00:00 xsawyerx
+    Version 3.10
+        * Fix binary artifacts from distribution.
+
+2018-04-21 16:49:00 xsawyerx
+    Version 3.09
+        * Fix "provides" in metadata (META.yml/META.json) to use the Storable
+          template instead of a small other file (which also didn't exist).
+
+2018-04-21 11:23:00 xsawyerx
+    Version 3.08
+        * (perl #132849) try to disable core files when deliberatly segfaulting.
+        * (perl #127743) don't probe Storable limits so much.
+        * (perl #132893) don't probe for Storable recursion limits on old Win32.
+        * (perl #132870) workaround VC2017 compiler bug.
+        * (perl #127743) re-work for debugging builds with MSVC.
+        * (perl #133039) dont build a Storable.so/.dll with a static perl build.
+
+2018-02-07 15:08:00 tonyc
+    Version 3.06
+
+        * support large object ids.  The code in theory supported arrays
+        with more than 2**32 elements, but references to the elements
+        emitted at the end of the array with be retrieved as references to
+        the wrong elements.
+        * 32-bit object ids over 2**31-1 weren't correctly handled.
+        * hook object id generation now supports 64-bit ids where needed
+        * writing 64-bit lengths in network order now works
+        * reading 64-bit lengths in network order now reads the components
+        in the correct order.
+        * retrieving large object tags are now only handled on 64-bit
+        platforms, large object tags should only be emitted for objects
+        that are too large for the 32-bit address space, so it was only
+        wasted code.
+        * reading 32-bit lengths for LSCALAR and LUTF8STR as unsigned
+        (perl #131990)
+        * reading flagged large object hashes didn't read the flags
+        * treat the 32-bit size of hook data as unsigned, values over 2GB
+        were treated as large (close to 2**64) parameters to NEWSV().
+        (perl #131999)
+        * added support for hook data over 4GB in size
+        * zero length data receievd from STORABLE_freeze() no longer
+        results in an invalid SV being passed to STORABLE_thaw/_attach()
+        (perl #118551)
+        * where practical, padding is now cleared when emitting a long
+        double (perl #131136)
+        * cache the value of $Storable::DEBUGME (since cperl enabled
+        Storable TRACEME builds for all -DDEBUGGING builds)
+        * no longer discard exceptions thrown by
+        STORABLE_freeze/_thaw/attach() (perl #25933)
+        * fix dependencies used to build Storable.pm from __Storable__.pm
+        * add experimental support for freezing/thawing regular
+        expressions (perl #50608)
+        * re-work recursion limiting to store the limit in a perl variable
+        instead of baked into Storable.$so.  This allows static Storable
+        builds to work, and avoids the kind of circular reference on
+        Storable.$so.
+
+2017-11-10 13:53:00 rurban
+    Version 3.05_17
+
+        * t/blessed.t: Better fix for pre-5.19.2 threaded perls
+        from p5p.
+
+2017-10-21 17:55:53 rurban
+    Version 3.05_16
+
+        * stacksize: larger stack reserve. Unified to 32 across all platforms.
+
+2017-10-21 10:08:53 rurban
+    Version 3.05_15
+
+        * t/blessed.t: TODO 5.10/5.12 threaded ref sv_true identity,
+        ref sv_true is not sv_true with those old perls.
+        * stacksize: low values only for Windows with unreliable SEGV signals,
+        back to higher depths with all others.
+
 2017-10-15 13:57:00 rurban
     Version 3.05_14