我的项目需要使用 mksqashfs 将一些东西打包到一个图像中，当内容发生变化时，我需要用新的替换图像文件。
在使用 sqashfs 之前，我使用了 tarball+md5sum。内容改变，md5改变。但是，mksquashfs 导出的图像文件每次都会变化，甚至内容都是一样的。
我正在遍历 mksquashfs 的所有参数，现在尝试在没有运气的情况下找到问题。
问问题
571 次
2 回答
0
github上还有另外一个修改过的mksquashfs：
https://github.com/dannyp11/squashfs4.3.git
这个 fork 添加了 -no-date 选项，使 mksquashfs 创建没有制作日期信息的 squash 文件。因此，如果内容保持不变，则结果 squash 文件具有相同的 chksum。
diff -ruNa squashfs4.3/README squashfs4.3-new/README
--- squashfs4.3/README 2014-05-12 13:17:59.000000000 -0700
+++ squashfs4.3-new/README 2017-08-01 23:05:44.428051586 -0700
@@ -127,6 +127,9 @@
-keep-as-directory if one source directory is specified, create a root
directory containing that directory, rather than the
contents of the directory
+-no-date do not store the date in the squash file
+ (-noappend and -no-fragments is also activated)
+ this way, the squash always has the same cksum
Filesystem filter options:
-p <pseudo-definition> Add pseudo file definition
diff -ruNa squashfs4.3/squashfs-tools/mksquashfs.c squashfs4.3-new/squashfs-tools/mksquashfs.c
--- squashfs4.3/squashfs-tools/mksquashfs.c 2014-05-12 15:18:20.000000000 -0700
+++ squashfs4.3-new/squashfs-tools/mksquashfs.c 2017-08-01 21:26:03.092224142 -0700
@@ -98,6 +98,7 @@
int use_regex = FALSE;
int nopad = FALSE;
int exit_on_error = FALSE;
+int no_date = FALSE;
long long global_uid = -1, global_gid = -1;
@@ -3139,7 +3140,7 @@
buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
buf.st_uid = getuid();
buf.st_gid = getgid();
- buf.st_mtime = time(NULL);
+ buf.st_mtime = (!no_date)? time(NULL) : 0;
buf.st_dev = 0;
buf.st_ino = 0;
dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0);
@@ -3526,7 +3527,7 @@
buf.st_gid = pseudo_ent->dev->gid;
buf.st_rdev = makedev(pseudo_ent->dev->major,
pseudo_ent->dev->minor);
- buf.st_mtime = time(NULL);
+ buf.st_mtime = (!no_date)? time(NULL) : 0;
buf.st_ino = pseudo_ino ++;
if(pseudo_ent->dev->type == 'd') {
@@ -5262,6 +5263,12 @@
else if(strcmp(argv[i], "-keep-as-directory") == 0)
keep_as_directory = TRUE;
+ else if (strcmp(argv[i], "-no-date") == 0) {
+ no_date = TRUE;
+ delete = TRUE;
+ no_fragments = TRUE;
+ }
+
else if(strcmp(argv[i], "-exit-on-error") == 0)
exit_on_error = TRUE;
@@ -5315,6 +5322,9 @@
ERROR("\t\t\tdirectory containing that directory, "
"rather than the\n");
ERROR("\t\t\tcontents of the directory\n");
+ ERROR("-no-date\t\tdo not store the date in the squash file\n");
+ ERROR("\t\t\t(-noappend and -no-fragments is also activated)\n");
+ ERROR("\t\t\tthis way, the squash always has the same cksum\n");
ERROR("\nFilesystem filter options:\n");
ERROR("-p <pseudo-definition>\tAdd pseudo file "
"definition\n");
@@ -5717,7 +5727,7 @@
sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
always_use_fragments, duplicate_checking, exportable,
no_xattrs, comp_opts);
- sBlk.mkfs_time = time(NULL);
+ sBlk.mkfs_time = (!no_date) ? time(NULL) : 0;
disable_info();
于 2017-09-18T07:25:48.393 回答
0
我有同样的问题，经过一番搜索，我发现了这个：
Here is a patch.
The patch adds "-deterministic" option.
Here is what the option does :
- it forces to one processor
- it forces the date globaly. The forced date is printed out and must be reused with "-forcedate" option in order to reproduced the same squashfs image.
- it sorts entries of directory as they are read.
- it delays the start of the fragment deflator thread. When this late start is active, the queue to fragment deflator thread is using an implementation of the queue with unlimited size.
Entries of directory must be sorted because the inode numbers are attributed as the files are discovered ("lookup_inode()" in "dirscan1()").
The format of the date after "-forcedate" is the number of seconds since the Epoch : the output of "date '+%s'".
With this patch, I can copy a tree of files, I can use tar or I can repack a mounted squashfs, and always have the same suqashfs file.
--- a/squashfs-tools/mksquashfs.c
+++ b/squashfs-tools/mksquashfs.c
@@ -113,6 +113,9 @@
#include "compressor.h"
#include "xattr.h"
+int forcedate = -1;
+int sort_dir_entry = 0;
+
int delete = FALSE;
int fd;
int cur_uncompressed = 0, estimated_uncompressed = 0;
@@ -342,6 +345,18 @@ struct queue {
void **data;
};
+/* struct describing queues used to pass data between threads */
+struct unlimited_queue {
+ struct unlimited_queue_node *first;
+ struct unlimited_queue_node *last;
+ pthread_mutex_t mutex;
+ pthread_cond_t any_event;
+};
+
+struct unlimited_queue_node {
+ void *data;
+ struct unlimited_queue_node* next;
+};
/* in memory uid tables */
#define ID_ENTRIES 256
@@ -362,6 +377,8 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
struct cache *reader_buffer, *writer_buffer, *fragment_buffer;
struct queue *to_reader, *from_reader, *to_writer, *from_writer, *from_deflate,
*to_frag;
+struct unlimited_queue *to_frag_unlimited = NULL;
+
pthread_t *thread, *deflator_thread, *frag_deflator_thread, progress_thread;
pthread_mutex_t fragment_mutex;
pthread_cond_t fragment_waiting;
@@ -488,6 +505,93 @@ void *queue_get(struct queue *queue)
return data;
}
+void queue_waitempty(struct queue *queue)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->readp != queue->writep)
+ pthread_cond_wait(&queue->full, &queue->mutex);
+
+ pthread_cond_signal(&queue->full);
+ pthread_mutex_unlock(&queue->mutex);
+}
+
+struct unlimited_queue *unlimited_queue_init()
+{
+ struct unlimited_queue *queue = malloc(sizeof(struct unlimited_queue));
+
+ if(queue == NULL)
+ goto failed;
+
+ queue->first = NULL;
+ queue->last = NULL;
+ pthread_mutex_init(&queue->mutex, NULL);
+ pthread_cond_init(&queue->any_event, NULL);
+
+ return queue;
+
+failed:
+ BAD_ERROR("Out of memory in queue_init\n");
+}
+
+void unlimited_queue_put(struct unlimited_queue *queue, void *data)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ struct unlimited_queue_node *newNode = malloc(sizeof(struct unlimited_queue_node));
+ if(newNode == NULL){
+ BAD_ERROR("Out of memory in queue_init\n");
+ }
+
+ newNode->data = data;
+ newNode->next = NULL;
+
+ pthread_mutex_unlock(&queue->mutex);
+
+ if(queue->first == NULL){
+ queue->first = newNode;
+ } else {
+ queue->last->next = newNode;
+ }
+ queue->last = newNode;
+ pthread_cond_broadcast(&queue->any_event);
+ pthread_mutex_unlock(&queue->mutex);
+}
+
+void *unlimited_queue_get(struct unlimited_queue *queue)
+{
+ void *data;
+
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->first == NULL)
+ pthread_cond_wait(&queue->any_event, &queue->mutex);
+
+ data = queue->first->data;
+ struct unlimited_queue_node* newFirst = queue->first->next;
+ free(queue->first);
+ queue->first = newFirst;
+/* eventually
+ if(newFirst == NULL){
+ queue->last == NULL;
+ }
+*/
+
+ pthread_cond_broadcast(&queue->any_event);
+ pthread_mutex_unlock(&queue->mutex);
+
+ return data;
+}
+
+void unlimited_queue_waitempty(struct unlimited_queue *queue)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->first != NULL)
+ pthread_cond_wait(&queue->any_event, &queue->mutex);
+
+ pthread_mutex_unlock(&queue->mutex);
+}
/* Cache status struct. Caches are used to keep
track of memory buffers passed between different threads */
@@ -1852,7 +1956,11 @@ void write_fragment()
fragment_data->block = fragments;
fragment_table[fragments].unused = 0;
fragments_outstanding ++;
- queue_put(to_frag, fragment_data);
+ if(to_frag_unlimited){
+ unlimited_queue_put(to_frag_unlimited, fragment_data);
+ }else{
+ queue_put(to_frag, fragment_data);
+ }
fragments ++;
fragment_size = 0;
pthread_mutex_unlock(&fragment_mutex);
@@ -2613,8 +2721,15 @@ void *frag_deflator(void *arg)
BAD_ERROR("frag_deflator:: compressor_init failed\n");
while(1) {
+ struct file_buffer *file_buffer = NULL;
+
+ if(to_frag_unlimited){
+ file_buffer = unlimited_queue_get(to_frag_unlimited);
+ }else{
+ file_buffer = queue_get(to_frag);
+ }
+
int c_byte, compressed_size;
- struct file_buffer *file_buffer = queue_get(to_frag);
struct file_buffer *write_buffer =
cache_get(writer_buffer, file_buffer->block +
FRAG_INDEX, 1);
@@ -3594,7 +3709,11 @@ void dir_scan(squashfs_inode *inode, char *pathname,
buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
buf.st_uid = getuid();
buf.st_gid = getgid();
- buf.st_mtime = time(NULL);
+ if(forcedate != -1){
+ buf.st_mtime = forcedate;
+ }else{
+ buf.st_mtime = time(NULL);
+ }
buf.st_dev = 0;
buf.st_ino = 0;
dir_ent->inode = lookup_inode(&buf);
@@ -3634,11 +3753,23 @@ void dir_scan(squashfs_inode *inode, char *pathname,
dir_ent->inode->type = SQUASHFS_DIR_TYPE;
}
+struct file_dir_name{
+ char* filename;
+ char* dirname;
+};
+
+int compare_file_dir_name(const void* a, const void* b){
+ return strcmp((*(struct file_dir_name**)a)->filename, (*(struct file_dir_name**)b)->filename);
+}
+
+#define BASE_ALLOC_INCREMENT 128
struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
int (_readdir)(char *, char *, struct dir_info *))
{
- char filename[8192], dir_name[8192];
+ char filename2[8192], dir_name2[8192];
+ char *filename = filename2;
+ char *dir_name = dir_name2;
struct dir_info *dir = scan1_opendir(pathname);
if(dir == NULL) {
@@ -3646,18 +3777,69 @@ struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
goto error;
}
- while(_readdir(filename, dir_name, dir) != FALSE) {
+ int base_count = 0;
+ void** base = NULL;
+
+ if(sort_dir_entry){
+ int base_allocated = BASE_ALLOC_INCREMENT;
+ base = malloc(base_allocated * sizeof(void*));
+
+ while(_readdir(filename2, dir_name2, dir) != FALSE) {
+ struct file_dir_name* new_element = malloc(sizeof(struct file_dir_name));
+ if(new_element == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+ new_element->filename = strdup(filename2);
+ if(new_element->filename == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+ new_element->dirname = strdup(dir_name2);
+ if(new_element->dirname == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+
+ if(base_count >= base_allocated){
+ base_allocated += BASE_ALLOC_INCREMENT;
+ base = realloc(base, base_allocated * sizeof(void*));
+ if(base == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+ }
+
+ base[base_count] = new_element;
+ ++base_count;
+ }
+
+ qsort(base, base_count, sizeof(void*), compare_file_dir_name);
+ }
+
+ int i = 0;
+
+ while(1){
+ if(sort_dir_entry){
+ if(i >= base_count){
+ break;
+ }
+ struct file_dir_name* cur_element = (struct file_dir_name*)base[i];
+ dir_name = cur_element->dirname;
+ filename = cur_element->filename;
+ }else{
+ if(_readdir(filename, dir_name, dir) == FALSE) {
+ break;
+ }
+ }
+
struct dir_info *sub_dir;
struct stat buf;
struct pathnames *new;
if(strcmp(dir_name, ".") == 0 || strcmp(dir_name, "..") == 0)
- continue;
+ goto loop_continue;
if(lstat(filename, &buf) == -1) {
ERROR("Cannot stat dir/file %s because %s, ignoring",
filename, strerror(errno));
- continue;
+ goto loop_continue;
}
if((buf.st_mode & S_IFMT) != S_IFREG &&
@@ -3669,29 +3851,40 @@ struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
(buf.st_mode & S_IFMT) != S_IFSOCK) {
ERROR("File %s has unrecognised filetype %d, ignoring"
"\n", filename, buf.st_mode & S_IFMT);
- continue;
+ goto loop_continue;
}
if(old_exclude) {
if(old_excluded(filename, &buf))
- continue;
+ goto loop_continue;
} else {
if(excluded(paths, dir_name, &new))
- continue;
+ goto loop_continue;
}
if((buf.st_mode & S_IFMT) == S_IFDIR) {
sub_dir = dir_scan1(filename, new, scan1_readdir);
if(sub_dir == NULL)
- continue;
+ goto loop_continue;
dir->directory_count ++;
} else
sub_dir = NULL;
add_dir_entry(dir_name, filename, sub_dir, lookup_inode(&buf),
dir);
+ loop_continue:
+ if(sort_dir_entry){
+ free(dir_name);
+ free(filename);
+ free(base[i]);
+ base[i] = NULL;
+ i++;
+ }
}
+ if(sort_dir_entry){
+ free(base);
+ }
scan1_freedir(dir);
error:
@@ -3777,7 +3970,11 @@ struct dir_info *dir_scan2(struct dir_info *dir, struct pseudo *pseudo)
buf.st_gid = pseudo_ent->dev->gid;
buf.st_rdev = makedev(pseudo_ent->dev->major,
pseudo_ent->dev->minor);
- buf.st_mtime = time(NULL);
+ if(forcedate != -1){
+ buf.st_mtime = forcedate;
+ }else{
+ buf.st_mtime = time(NULL);
+ }
buf.st_ino = pseudo_ino ++;
if(pseudo_ent->dev->type == 'f') {
@@ -4039,6 +4236,7 @@ void add_old_root_entry(char *name, squashfs_inode inode, int inode_number,
old_root_entry[old_root_entries++].inode.root_entry = TRUE;
}
+int late_fragment_thread_start = 0;
void initialise_threads(int readb_mbytes, int writeb_mbytes,
int fragmentb_mbytes)
@@ -4046,7 +4244,7 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
int i;
sigset_t sigmask, old_mask;
int reader_buffer_size = readb_mbytes << (20 - block_log);
- int fragment_buffer_size = fragmentb_mbytes << (20 - block_log);
+ int fragment_buffer_size = (fragmentb_mbytes == -1 ? -1 : fragmentb_mbytes << (20 - block_log));
/*
* writer_buffer_size is global because it is needed in
@@ -4095,10 +4293,16 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
to_writer = queue_init(writer_buffer_size);
from_writer = queue_init(1);
from_deflate = queue_init(reader_buffer_size);
- to_frag = queue_init(fragment_buffer_size);
+
+ if(fragment_buffer_size == -1){
+ to_frag_unlimited = unlimited_queue_init();
+ }else{
+ to_frag = queue_init(fragment_buffer_size);
+ }
+
reader_buffer = cache_init(block_size, reader_buffer_size);
writer_buffer = cache_init(block_size, writer_buffer_size);
- fragment_buffer = cache_init(block_size, fragment_buffer_size);
+ fragment_buffer = cache_init(block_size, fragment_buffer_size != -1 ? fragment_buffer_size : FRAGMENT_BUFFER_DEFAULT << (20 - block_log));
pthread_create(&thread[0], NULL, reader, NULL);
pthread_create(&thread[1], NULL, writer, NULL);
pthread_create(&progress_thread, NULL, progress_thrd, NULL);
@@ -4109,9 +4313,11 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
if(pthread_create(&deflator_thread[i], NULL, deflator, NULL) !=
0)
BAD_ERROR("Failed to create thread\n");
- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
- NULL) != 0)
- BAD_ERROR("Failed to create thread\n");
+ if(!late_fragment_thread_start){
+ if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
+ NULL) != 0)
+ BAD_ERROR("Failed to create thread\n");
+ }
}
printf("Parallel mksquashfs: Using %d processor%s\n", processors,
@@ -4701,6 +4907,31 @@ int main(int argc, char *argv[])
"megabyte or larger\n", argv[0]);
exit(1);
}
+ } else if(strcmp(argv[i], "-forcedate") == 0) {
+ if(++i == argc) {
+ ERROR("%s: -forcedate missing fake date\n", argv[0]);
+ exit(1);
+ }
+ forcedate = strtol(argv[i], &b, 10);
+ if(*b != '\0') {
+ ERROR("%s: -forcedate invalid parameter\n", argv[0]);
+ exit(1);
+ }
+ } else if(strcmp(argv[i], "-deterministic") == 0) {
+ if(forcedate == -1){
+ forcedate = time(NULL);
+ }
+ processors = 1;
+ late_fragment_thread_start = 1;
+ fragmentb_mbytes = -1;
+ if(sort_dir_entry == 0){
+ sort_dir_entry = 1;
+ }
+ } else if(strcmp(argv[i], "-late-fragment-thread-start") == 0) {
+ late_fragment_thread_start = 1;
+ fragmentb_mbytes = -1;
+ } else if(strcmp(argv[i], "-unlimited-fragment-queue") == 0) {
+ fragmentb_mbytes = -1;
} else if(strcmp(argv[i], "-b") == 0) {
if(++i == argc) {
ERROR("%s: -b missing block size\n", argv[0]);
@@ -5292,7 +5523,11 @@ printOptions:
sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
always_use_fragments, duplicate_checking, exportable,
no_xattrs, comp_opts);
- sBlk.mkfs_time = time(NULL);
+ if(forcedate != -1){
+ sBlk.mkfs_time = forcedate;
+ } else {
+ sBlk.mkfs_time = time(NULL);
+ }
restore_filesystem:
if(progress && estimated_uncompressed) {
@@ -5301,6 +5536,14 @@ restore_filesystem:
}
write_fragment();
+
+ if(late_fragment_thread_start){
+ queue_waitempty(from_reader);
+ if(pthread_create(&frag_deflator_thread[0], NULL, frag_deflator,
+ NULL) != 0)
+ BAD_ERROR("Failed to create thread\n");
+ }
+
sBlk.fragments = fragments;
if(!restoring) {
unlock_fragments();
@@ -5340,7 +5583,8 @@ restore_filesystem:
write_destination(fd, SQUASHFS_START, sizeof(sBlk), &sBlk);
if(!nopad && (i = bytes & (4096 - 1))) {
- char temp[4096] = {0};
+ char temp[4096];
+ memset(&temp, 0, 4096);
write_destination(fd, bytes, 4096 - i, temp);
}
@@ -5401,6 +5645,9 @@ restore_filesystem:
printf("Number of directories %d\n", dir_count);
printf("Number of ids (unique uids + gids) %d\n", id_count);
printf("Number of uids %d\n", uid_count);
+ if(forcedate != -1){
+ printf("forcedate was %d\n", forcedate);
+ }
for(i = 0; i < id_count; i++) {
if(id_table[i]->flags & ISA_UID) {
于 2017-09-18T06:14:06.830 回答