Android/android-simg2img
1
0
Fork 0
Code Pull Requests Actions Releases Activity

Android-10 Support

Browse Source 
Pulled latest libsparse sources from upstream. AOSP rewrote entire toolkit to CPP
with optimisations in various areas. Backwards compatibility for older Android
versions not tested yet.

Signed-off-by: Anestis Bechtsoudis <anestis@census-labs.com>
This commit is contained in:
Anestis Bechtsoudis 2019-11-05 11:33:57 +02:00
parent 223e415fb3
commit 7ab241c783
No known key found for this signature in database
GPG Key ID: D7248CD88CBAB8EE
33 changed files with 2819 additions and 2818 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
LICENSE
View File

@ -1,14 +0,0 @@
Copyright (C) 2010 The Android Open Source Project
Modifications copyright (C) 2014 Anestis Bechtsoudis
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

63
Makefile
View File

@ -15,26 +15,27 @@
#
PREFIX ?= /usr/local
CC ?= gcc
LD ?= gcc
DEP_CC ?= gcc
AR ?= ar
RANLIB ?= ranlib
STRIP ?= strip
CFLAGS += -O2 -Wall -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE=1
CXX ?= g++
LD ?= g++
DEP_CXX ?= g++
AR ?= ar
RANLIB ?= ranlib
STRIP ?= strip
CPPFLAGS += -std=gnu++17 -O2 -W -Wall -Werror -Wextra
# libsparse
LIB_NAME = sparse
SLIB = lib$(LIB_NAME).a
LIB_SRCS = \
backed_block.c \
output_file.c \
sparse.c \
sparse_crc32.c \
sparse_err.c \
sparse_read.c
LIB_OBJS = $(LIB_SRCS:%.c=%.o)
LIB_INCS = -Iinclude
backed_block.cpp \
output_file.cpp \
sparse.cpp \
sparse_crc32.cpp \
sparse_err.cpp \
sparse_read.cpp \
android-base/stringprintf.cpp
LIB_OBJS = $(LIB_SRCS:%.cpp=%.o)
LIB_INCS = -Iinclude -Iandroid-base/include
LDFLAGS += -L. -l$(LIB_NAME) -lm -lz
@ -42,20 +43,20 @@ BINS = simg2img simg2simg img2simg append2simg
HEADERS = include/sparse/sparse.h
# simg2img
SIMG2IMG_SRCS = simg2img.c
SIMG2IMG_OBJS = $(SIMG2IMG_SRCS:%.c=%.o)
SIMG2IMG_SRCS = simg2img.cpp
SIMG2IMG_OBJS = $(SIMG2IMG_SRCS:%.cpp=%.o)
# simg2simg
SIMG2SIMG_SRCS = simg2simg.c
SIMG2SIMG_OBJS = $(SIMG2SIMG_SRCS:%.c=%.o)
SIMG2SIMG_SRCS = simg2simg.cpp
SIMG2SIMG_OBJS = $(SIMG2SIMG_SRCS:%.cpp=%.o)
# img2simg
IMG2SIMG_SRCS = $(LIBSPARSE_SRCS) img2simg.c
IMG2SIMG_OBJS = $(IMG2SIMG_SRCS:%.c=%.o)
IMG2SIMG_SRCS = img2simg.cpp
IMG2SIMG_OBJS = $(IMG2SIMG_SRCS:%.cpp=%.o)
# append2simg
APPEND2SIMG_SRCS = $(LIBSPARSE_SRCS) append2simg.c
APPEND2SIMG_OBJS = $(APPEND2SIMG_SRCS:%.c=%.o)
APPEND2SIMG_SRCS = append2simg.cpp
APPEND2SIMG_OBJS = $(APPEND2SIMG_SRCS:%.cpp=%.o)
SRCS = \
$(SIMG2IMG_SRCS) \
@ -80,19 +81,19 @@ $(LIB_NAME): $(LIB_OBJS)
$(RANLIB) $(SLIB)
simg2img: $(SIMG2IMG_SRCS) $(LIB_NAME)
$(CC) $(CFLAGS) $(LIB_INCS) -o simg2img $< $(LDFLAGS)
$(CXX) $(CPPFLAGS) $(LIB_INCS) -o simg2img $< $(LDFLAGS)
simg2simg: $(SIMG2SIMG_SRCS) $(LIB_NAME)
$(CC) $(CFLAGS) $(LIB_INCS) -o simg2simg $< $(LDFLAGS)
$(CXX) $(CPPFLAGS) $(LIB_INCS) -o simg2simg $< $(LDFLAGS)
img2simg: $(IMG2SIMG_SRCS) $(LIB_NAME)
$(CC) $(CFLAGS) $(LIB_INCS) -o img2simg $< $(LDFLAGS)
$(CXX) $(CPPFLAGS) $(LIB_INCS) -o img2simg $< $(LDFLAGS)
append2simg: $(APPEND2SIMG_SRCS) $(LIB_NAME)
$(CC) $(CFLAGS) $(LIB_INCS) -o append2simg $< $(LDFLAGS)
$(CXX) $(CPPFLAGS) $(LIB_INCS) -o append2simg $< $(LDFLAGS)
%.o: %.c .depend
$(CC) -c $(CFLAGS) $(LIB_INCS) $< -o $@
%.o: %.cpp .depend
$(CXX) $(CPPFLAGS) $(LIB_INCS) -c $< -o $@
clean:
$(RM) -f *.o *.a simg2img simg2simg img2simg append2simg .depend
@ -103,7 +104,7 @@ endif
.depend:
@$(RM) .depend
@$(foreach SRC, $(SRCS), $(DEP_CC) $(LIB_INCS) $(SRC) $(CFLAGS) -MT $(SRC:%.c=%.o) -MM >> .depend;)
@$(foreach SRC, $(SRCS), $(DEP_CXX) $(LIB_INCS) $(SRC) $(CFLAGS) -MT $(SRC:%.c=%.o) -MM >> .depend;)
indent:
indent -linux -l100 -lc100 -nut -i4 *.c *.h; rm -f *~
indent -linux -l100 -lc100 -nut -i4 *.cpp *.h; rm -f *~

1
OWNERS Normal file
View File

@ -0,0 +1 @@
ccross@google.com

40
android-base/include/android-base/stringprintf.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <stdarg.h>
#include <string>
namespace android {
namespace base {
// These printf-like functions are implemented in terms of vsnprintf, so they
// use the same attribute for compile-time format string checking.
// Returns a string corresponding to printf-like formatting of the arguments.
std::string StringPrintf(const char* fmt, ...) __attribute__((__format__(__printf__, 1, 2)));
// Appends a printf-like formatting of the arguments to 'dst'.
void StringAppendF(std::string* dst, const char* fmt, ...)
__attribute__((__format__(__printf__, 2, 3)));
// Appends a printf-like formatting of the arguments to 'dst'.
void StringAppendV(std::string* dst, const char* format, va_list ap)
__attribute__((__format__(__printf__, 2, 0)));
} // namespace base
} // namespace android

85
android-base/stringprintf.cpp Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "android-base/stringprintf.h"
#include <stdio.h>
#include <string>
namespace android {
namespace base {
void StringAppendV(std::string* dst, const char* format, va_list ap) {
// First try with a small fixed size buffer
char space[1024];
// It's possible for methods that use a va_list to invalidate
// the data in it upon use. The fix is to make a copy
// of the structure before using it and use that copy instead.
va_list backup_ap;
va_copy(backup_ap, ap);
int result = vsnprintf(space, sizeof(space), format, backup_ap);
va_end(backup_ap);
if (result < static_cast<int>(sizeof(space))) {
if (result >= 0) {
// Normal case -- everything fit.
dst->append(space, result);
return;
}
if (result < 0) {
// Just an error.
return;
}
}
// Increase the buffer size to the size requested by vsnprintf,
// plus one for the closing \0.
int length = result + 1;
char* buf = new char[length];
// Restore the va_list before we use it again
va_copy(backup_ap, ap);
result = vsnprintf(buf, length, format, backup_ap);
va_end(backup_ap);
if (result >= 0 && result < length) {
// It fit
dst->append(buf, result);
}
delete[] buf;
}
std::string StringPrintf(const char* fmt, ...) {
va_list ap;
va_start(ap, fmt);
std::string result;
StringAppendV(&result, fmt, ap);
va_end(ap);
return result;
}
void StringAppendF(std::string* dst, const char* format, ...) {
va_list ap;
va_start(ap, format);
StringAppendV(dst, format, ap);
va_end(ap);
}
} // namespace base
} // namespace android

140
append2simg.c
View File

@ -1,140 +0,0 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sparse/sparse.h>
#include "sparse_file.h"
#include "backed_block.h"
#ifndef O_BINARY
#define O_BINARY 0
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t
#endif
void usage()
{
fprintf(stderr, "Usage: append2simg <output> <input>\n");
}
int main(int argc, char *argv[])
{
int output;
int output_block;
char *output_path;
struct sparse_file *sparse_output;
int input;
char *input_path;
off64_t input_len;
int tmp_fd;
char *tmp_path;
int ret;
if (argc == 3) {
output_path = argv[1];
input_path = argv[2];
} else {
usage();
exit(-1);
}
ret = asprintf(&tmp_path, "%s.append2simg", output_path);
if (ret < 0) {
fprintf(stderr, "Couldn't allocate filename\n");
exit(-1);
}
output = open(output_path, O_RDWR | O_BINARY);
if (output < 0) {
fprintf(stderr, "Couldn't open output file (%s)\n", strerror(errno));
exit(-1);
}
sparse_output = sparse_file_import_auto(output, false, true);
if (!sparse_output) {
fprintf(stderr, "Couldn't import output file\n");
exit(-1);
}
input = open(input_path, O_RDONLY | O_BINARY);
if (input < 0) {
fprintf(stderr, "Couldn't open input file (%s)\n", strerror(errno));
exit(-1);
}
input_len = lseek64(input, 0, SEEK_END);
if (input_len < 0) {
fprintf(stderr, "Couldn't get input file length (%s)\n", strerror(errno));
exit(-1);
} else if (input_len % sparse_output->block_size) {
fprintf(stderr, "Input file is not a multiple of the output file's block size");
exit(-1);
}
lseek64(input, 0, SEEK_SET);
output_block = sparse_output->len / sparse_output->block_size;
if (sparse_file_add_fd(sparse_output, input, 0, input_len, output_block) < 0) {
fprintf(stderr, "Couldn't add input file\n");
exit(-1);
}
sparse_output->len += input_len;
tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_BINARY, 0664);
if (tmp_fd < 0) {
fprintf(stderr, "Couldn't open temporary file (%s)\n", strerror(errno));
exit(-1);
}
lseek64(output, 0, SEEK_SET);
if (sparse_file_write(sparse_output, tmp_fd, false, true, false) < 0) {
fprintf(stderr, "Failed to write sparse file\n");
exit(-1);
}
sparse_file_destroy(sparse_output);
close(tmp_fd);
close(output);
close(input);
ret = rename(tmp_path, output_path);
if (ret < 0) {
fprintf(stderr, "Failed to rename temporary file (%s)\n", strerror(errno));
exit(-1);
}
free(tmp_path);
exit(0);
}

137
append2simg.cpp Normal file
View File

@ -0,0 +1,137 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sparse/sparse.h>
#include "backed_block.h"
#include "sparse_file.h"
#ifndef O_BINARY
#define O_BINARY 0
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t
#endif
void usage() {
fprintf(stderr, "Usage: append2simg <output> <input>\n");
}
int main(int argc, char* argv[]) {
int output;
int output_block;
char* output_path;
struct sparse_file* sparse_output;
int input;
char* input_path;
off64_t input_len;
int tmp_fd;
char* tmp_path;
int ret;
if (argc == 3) {
output_path = argv[1];
input_path = argv[2];
} else {
usage();
exit(-1);
}
ret = asprintf(&tmp_path, "%s.append2simg", output_path);
if (ret < 0) {
fprintf(stderr, "Couldn't allocate filename\n");
exit(-1);
}
output = open(output_path, O_RDWR | O_BINARY);
if (output < 0) {
fprintf(stderr, "Couldn't open output file (%s)\n", strerror(errno));
exit(-1);
}
sparse_output = sparse_file_import_auto(output, false, true);
if (!sparse_output) {
fprintf(stderr, "Couldn't import output file\n");
exit(-1);
}
input = open(input_path, O_RDONLY | O_BINARY);
if (input < 0) {
fprintf(stderr, "Couldn't open input file (%s)\n", strerror(errno));
exit(-1);
}
input_len = lseek64(input, 0, SEEK_END);
if (input_len < 0) {
fprintf(stderr, "Couldn't get input file length (%s)\n", strerror(errno));
exit(-1);
} else if (input_len % sparse_output->block_size) {
fprintf(stderr, "Input file is not a multiple of the output file's block size");
exit(-1);
}
lseek64(input, 0, SEEK_SET);
output_block = sparse_output->len / sparse_output->block_size;
if (sparse_file_add_fd(sparse_output, input, 0, input_len, output_block) < 0) {
fprintf(stderr, "Couldn't add input file\n");
exit(-1);
}
sparse_output->len += input_len;
tmp_fd = open(tmp_path, O_WRONLY | O_CREAT | O_BINARY, 0664);
if (tmp_fd < 0) {
fprintf(stderr, "Couldn't open temporary file (%s)\n", strerror(errno));
exit(-1);
}
lseek64(output, 0, SEEK_SET);
if (sparse_file_write(sparse_output, tmp_fd, false, true, false) < 0) {
fprintf(stderr, "Failed to write sparse file\n");
exit(-1);
}
sparse_file_destroy(sparse_output);
close(tmp_fd);
close(output);
close(input);
ret = rename(tmp_path, output_path);
if (ret < 0) {
fprintf(stderr, "Failed to rename temporary file (%s)\n", strerror(errno));
exit(-1);
}
free(tmp_path);
exit(0);
}

398
backed_block.c
View File

@ -1,398 +0,0 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <assert.h>
#include <errno.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "backed_block.h"
#include "sparse_defs.h"
struct backed_block {
unsigned int block;
unsigned int len;
enum backed_block_type type;
union {
struct {
void *data;
} data;
struct {
char *filename;
int64_t offset;
} file;
struct {
int fd;
int64_t offset;
} fd;
struct {
uint32_t val;
} fill;
};
struct backed_block *next;
};
struct backed_block_list {
struct backed_block *data_blocks;
struct backed_block *last_used;
unsigned int block_size;
};
struct backed_block *backed_block_iter_new(struct backed_block_list *bbl)
{
return bbl->data_blocks;
}
struct backed_block *backed_block_iter_next(struct backed_block *bb)
{
return bb->next;
}
unsigned int backed_block_len(struct backed_block *bb)
{
return bb->len;
}
unsigned int backed_block_block(struct backed_block *bb)
{
return bb->block;
}
void *backed_block_data(struct backed_block *bb)
{
assert(bb->type == BACKED_BLOCK_DATA);
return bb->data.data;
}
const char *backed_block_filename(struct backed_block *bb)
{
assert(bb->type == BACKED_BLOCK_FILE);
return bb->file.filename;
}
int backed_block_fd(struct backed_block *bb)
{
assert(bb->type == BACKED_BLOCK_FD);
return bb->fd.fd;
}
int64_t backed_block_file_offset(struct backed_block * bb)
{
assert(bb->type == BACKED_BLOCK_FILE || bb->type == BACKED_BLOCK_FD);
if (bb->type == BACKED_BLOCK_FILE) {
return bb->file.offset;
} else { /* bb->type == BACKED_BLOCK_FD */
return bb->fd.offset;
}
}
uint32_t backed_block_fill_val(struct backed_block * bb)
{
assert(bb->type == BACKED_BLOCK_FILL);
return bb->fill.val;
}
enum backed_block_type backed_block_type(struct backed_block *bb)
{
return bb->type;
}
void backed_block_destroy(struct backed_block *bb)
{
if (bb->type == BACKED_BLOCK_FILE) {
free(bb->file.filename);
}
free(bb);
}
struct backed_block_list *backed_block_list_new(unsigned int block_size)
{
struct backed_block_list *b = calloc(sizeof(struct backed_block_list), 1);
b->block_size = block_size;
return b;
}
void backed_block_list_destroy(struct backed_block_list *bbl)
{
if (bbl->data_blocks) {
struct backed_block *bb = bbl->data_blocks;
while (bb) {
struct backed_block *next = bb->next;
backed_block_destroy(bb);
bb = next;
}
}
free(bbl);
}
void backed_block_list_move(struct backed_block_list *from,
struct backed_block_list *to, struct backed_block *start,
struct backed_block *end)
{
struct backed_block *bb;
if (start == NULL) {
start = from->data_blocks;
}
if (!end) {
for (end = start; end && end->next; end = end->next) ;
}
if (start == NULL || end == NULL) {
return;
}
from->last_used = NULL;
to->last_used = NULL;
if (from->data_blocks == start) {
from->data_blocks = end->next;
} else {
for (bb = from->data_blocks; bb; bb = bb->next) {
if (bb->next == start) {
bb->next = end->next;
break;
}
}
}
if (!to->data_blocks) {
to->data_blocks = start;
end->next = NULL;
} else {
for (bb = to->data_blocks; bb; bb = bb->next) {
if (!bb->next || bb->next->block > start->block) {
end->next = bb->next;
bb->next = start;
break;
}
}
}
}
/* may free b */
static int merge_bb(struct backed_block_list *bbl, struct backed_block *a, struct backed_block *b)
{
unsigned int block_len;
/* Block doesn't exist (possible if one block is the last block) */
if (!a || !b) {
return -EINVAL;
}
assert(a->block < b->block);
/* Blocks are of different types */
if (a->type != b->type) {
return -EINVAL;
}
/* Blocks are not adjacent */
block_len = a->len / bbl->block_size; /* rounds down */
if (a->block + block_len != b->block) {
return -EINVAL;
}
switch (a->type) {
case BACKED_BLOCK_DATA:
/* Don't support merging data for now */
return -EINVAL;
case BACKED_BLOCK_FILL:
if (a->fill.val != b->fill.val) {
return -EINVAL;
}
break;
case BACKED_BLOCK_FILE:
/* Already make sure b->type is BACKED_BLOCK_FILE */
if (strcmp(a->file.filename, b->file.filename) || a->file.offset + a->len != b->file.offset) {
return -EINVAL;
}
break;
case BACKED_BLOCK_FD:
if (a->fd.fd != b->fd.fd || a->fd.offset + a->len != b->fd.offset) {
return -EINVAL;
}
break;
}
/* Blocks are compatible and adjacent, with a before b. Merge b into a,
* and free b */
a->len += b->len;
a->next = b->next;
backed_block_destroy(b);
return 0;
}
static int queue_bb(struct backed_block_list *bbl, struct backed_block *new_bb)
{
struct backed_block *bb;
if (bbl->data_blocks == NULL) {
bbl->data_blocks = new_bb;
return 0;
}
if (bbl->data_blocks->block > new_bb->block) {
new_bb->next = bbl->data_blocks;
bbl->data_blocks = new_bb;
return 0;
}
/* Optimization: blocks are mostly queued in sequence, so save the
pointer to the last bb that was added, and start searching from
there if the next block number is higher */
if (bbl->last_used && new_bb->block > bbl->last_used->block)
bb = bbl->last_used;
else
bb = bbl->data_blocks;
bbl->last_used = new_bb;
for (; bb->next && bb->next->block < new_bb->block; bb = bb->next) ;
if (bb->next == NULL) {
bb->next = new_bb;
} else {
new_bb->next = bb->next;
bb->next = new_bb;
}
merge_bb(bbl, new_bb, new_bb->next);
if (!merge_bb(bbl, bb, new_bb)) {
/* new_bb destroyed, point to retained as last_used */
bbl->last_used = bb;
}
return 0;
}
/* Queues a fill block of memory to be written to the specified data blocks */
int backed_block_add_fill(struct backed_block_list *bbl, unsigned int fill_val,
unsigned int len, unsigned int block)
{
struct backed_block *bb = calloc(1, sizeof(struct backed_block));
if (bb == NULL) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_FILL;
bb->fill.val = fill_val;
bb->next = NULL;
return queue_bb(bbl, bb);
}
/* Queues a block of memory to be written to the specified data blocks */
int backed_block_add_data(struct backed_block_list *bbl, void *data,
unsigned int len, unsigned int block)
{
struct backed_block *bb = calloc(1, sizeof(struct backed_block));
if (bb == NULL) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_DATA;
bb->data.data = data;
bb->next = NULL;
return queue_bb(bbl, bb);
}
/* Queues a chunk of a file on disk to be written to the specified data blocks */
int backed_block_add_file(struct backed_block_list *bbl, const char *filename,
int64_t offset, unsigned int len, unsigned int block)
{
struct backed_block *bb = calloc(1, sizeof(struct backed_block));
if (bb == NULL) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_FILE;
bb->file.filename = strdup(filename);
bb->file.offset = offset;
bb->next = NULL;
return queue_bb(bbl, bb);
}
/* Queues a chunk of a fd to be written to the specified data blocks */
int backed_block_add_fd(struct backed_block_list *bbl, int fd, int64_t offset,
unsigned int len, unsigned int block)
{
struct backed_block *bb = calloc(1, sizeof(struct backed_block));
if (bb == NULL) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_FD;
bb->fd.fd = fd;
bb->fd.offset = offset;
bb->next = NULL;
return queue_bb(bbl, bb);
}
int backed_block_split(struct backed_block_list *bbl, struct backed_block *bb, unsigned int max_len)
{
struct backed_block *new_bb;
max_len = ALIGN_DOWN(max_len, bbl->block_size);
if (bb->len <= max_len) {
return 0;
}
new_bb = malloc(sizeof(struct backed_block));
if (new_bb == NULL) {
return -ENOMEM;
}
*new_bb = *bb;
new_bb->len = bb->len - max_len;
new_bb->block = bb->block + max_len / bbl->block_size;
new_bb->next = bb->next;
bb->next = new_bb;
bb->len = max_len;
switch (bb->type) {
case BACKED_BLOCK_DATA:
new_bb->data.data = (char *)bb->data.data + max_len;
break;
case BACKED_BLOCK_FILE:
new_bb->file.offset += max_len;
break;
case BACKED_BLOCK_FD:
new_bb->fd.offset += max_len;
break;
case BACKED_BLOCK_FILL:
break;
}
return 0;
}

380
backed_block.cpp Normal file
View File

@ -0,0 +1,380 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <assert.h>
#include <errno.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "backed_block.h"
#include "sparse_defs.h"
struct backed_block {
unsigned int block;
unsigned int len;
enum backed_block_type type;
union {
struct {
void* data;
} data;
struct {
char* filename;
int64_t offset;
} file;
struct {
int fd;
int64_t offset;
} fd;
struct {
uint32_t val;
} fill;
};
struct backed_block* next;
};
struct backed_block_list {
struct backed_block* data_blocks;
struct backed_block* last_used;
unsigned int block_size;
};
struct backed_block* backed_block_iter_new(struct backed_block_list* bbl) {
return bbl->data_blocks;
}
struct backed_block* backed_block_iter_next(struct backed_block* bb) {
return bb->next;
}
unsigned int backed_block_len(struct backed_block* bb) {
return bb->len;
}
unsigned int backed_block_block(struct backed_block* bb) {
return bb->block;
}
void* backed_block_data(struct backed_block* bb) {
assert(bb->type == BACKED_BLOCK_DATA);
return bb->data.data;
}
const char* backed_block_filename(struct backed_block* bb) {
assert(bb->type == BACKED_BLOCK_FILE);
return bb->file.filename;
}
int backed_block_fd(struct backed_block* bb) {
assert(bb->type == BACKED_BLOCK_FD);
return bb->fd.fd;
}
int64_t backed_block_file_offset(struct backed_block* bb) {
assert(bb->type == BACKED_BLOCK_FILE || bb->type == BACKED_BLOCK_FD);
if (bb->type == BACKED_BLOCK_FILE) {
return bb->file.offset;
} else { /* bb->type == BACKED_BLOCK_FD */
return bb->fd.offset;
}
}
uint32_t backed_block_fill_val(struct backed_block* bb) {
assert(bb->type == BACKED_BLOCK_FILL);
return bb->fill.val;
}
enum backed_block_type backed_block_type(struct backed_block* bb) {
return bb->type;
}
void backed_block_destroy(struct backed_block* bb) {
if (bb->type == BACKED_BLOCK_FILE) {
free(bb->file.filename);
}
free(bb);
}
struct backed_block_list* backed_block_list_new(unsigned int block_size) {
struct backed_block_list* b =
reinterpret_cast<backed_block_list*>(calloc(sizeof(struct backed_block_list), 1));
b->block_size = block_size;
return b;
}
void backed_block_list_destroy(struct backed_block_list* bbl) {
if (bbl->data_blocks) {
struct backed_block* bb = bbl->data_blocks;
while (bb) {
struct backed_block* next = bb->next;
backed_block_destroy(bb);
bb = next;
}
}
free(bbl);
}
void backed_block_list_move(struct backed_block_list* from, struct backed_block_list* to,
struct backed_block* start, struct backed_block* end) {
struct backed_block* bb;
if (start == nullptr) {
start = from->data_blocks;
}
if (!end) {
for (end = start; end && end->next; end = end->next)
;
}
if (start == nullptr || end == nullptr) {
return;
}
from->last_used = nullptr;
to->last_used = nullptr;
if (from->data_blocks == start) {
from->data_blocks = end->next;
} else {
for (bb = from->data_blocks; bb; bb = bb->next) {
if (bb->next == start) {
bb->next = end->next;
break;
}
}
}
if (!to->data_blocks) {
to->data_blocks = start;
end->next = nullptr;
} else {
for (bb = to->data_blocks; bb; bb = bb->next) {
if (!bb->next || bb->next->block > start->block) {
end->next = bb->next;
bb->next = start;
break;
}
}
}
}
/* may free b */
static int merge_bb(struct backed_block_list* bbl, struct backed_block* a, struct backed_block* b) {
unsigned int block_len;
/* Block doesn't exist (possible if one block is the last block) */
if (!a || !b) {
return -EINVAL;
}
assert(a->block < b->block);
/* Blocks are of different types */
if (a->type != b->type) {
return -EINVAL;
}
/* Blocks are not adjacent */
block_len = a->len / bbl->block_size; /* rounds down */
if (a->block + block_len != b->block) {
return -EINVAL;
}
switch (a->type) {
case BACKED_BLOCK_DATA:
/* Don't support merging data for now */
return -EINVAL;
case BACKED_BLOCK_FILL:
if (a->fill.val != b->fill.val) {
return -EINVAL;
}
break;
case BACKED_BLOCK_FILE:
/* Already make sure b->type is BACKED_BLOCK_FILE */
if (strcmp(a->file.filename, b->file.filename) || a->file.offset + a->len != b->file.offset) {
return -EINVAL;
}
break;
case BACKED_BLOCK_FD:
if (a->fd.fd != b->fd.fd || a->fd.offset + a->len != b->fd.offset) {
return -EINVAL;
}
break;
}
/* Blocks are compatible and adjacent, with a before b. Merge b into a,
* and free b */
a->len += b->len;
a->next = b->next;
backed_block_destroy(b);
return 0;
}
static int queue_bb(struct backed_block_list* bbl, struct backed_block* new_bb) {
struct backed_block* bb;
if (bbl->data_blocks == nullptr) {
bbl->data_blocks = new_bb;
return 0;
}
if (bbl->data_blocks->block > new_bb->block) {
new_bb->next = bbl->data_blocks;
bbl->data_blocks = new_bb;
return 0;
}
/* Optimization: blocks are mostly queued in sequence, so save the
pointer to the last bb that was added, and start searching from
there if the next block number is higher */
if (bbl->last_used && new_bb->block > bbl->last_used->block)
bb = bbl->last_used;
else
bb = bbl->data_blocks;
bbl->last_used = new_bb;
for (; bb->next && bb->next->block < new_bb->block; bb = bb->next)
;
if (bb->next == nullptr) {
bb->next = new_bb;
} else {
new_bb->next = bb->next;
bb->next = new_bb;
}
merge_bb(bbl, new_bb, new_bb->next);
if (!merge_bb(bbl, bb, new_bb)) {
/* new_bb destroyed, point to retained as last_used */
bbl->last_used = bb;
}
return 0;
}
/* Queues a fill block of memory to be written to the specified data blocks */
int backed_block_add_fill(struct backed_block_list* bbl, unsigned int fill_val, unsigned int len,
unsigned int block) {
struct backed_block* bb = reinterpret_cast<backed_block*>(calloc(1, sizeof(struct backed_block)));
if (bb == nullptr) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_FILL;
bb->fill.val = fill_val;
bb->next = nullptr;
return queue_bb(bbl, bb);
}
/* Queues a block of memory to be written to the specified data blocks */
int backed_block_add_data(struct backed_block_list* bbl, void* data, unsigned int len,
unsigned int block) {
struct backed_block* bb = reinterpret_cast<backed_block*>(calloc(1, sizeof(struct backed_block)));
if (bb == nullptr) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_DATA;
bb->data.data = data;
bb->next = nullptr;
return queue_bb(bbl, bb);
}
/* Queues a chunk of a file on disk to be written to the specified data blocks */
int backed_block_add_file(struct backed_block_list* bbl, const char* filename, int64_t offset,
unsigned int len, unsigned int block) {
struct backed_block* bb = reinterpret_cast<backed_block*>(calloc(1, sizeof(struct backed_block)));
if (bb == nullptr) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_FILE;
bb->file.filename = strdup(filename);
bb->file.offset = offset;
bb->next = nullptr;
return queue_bb(bbl, bb);
}
/* Queues a chunk of a fd to be written to the specified data blocks */
int backed_block_add_fd(struct backed_block_list* bbl, int fd, int64_t offset, unsigned int len,
unsigned int block) {
struct backed_block* bb = reinterpret_cast<backed_block*>(calloc(1, sizeof(struct backed_block)));
if (bb == nullptr) {
return -ENOMEM;
}
bb->block = block;
bb->len = len;
bb->type = BACKED_BLOCK_FD;
bb->fd.fd = fd;
bb->fd.offset = offset;
bb->next = nullptr;
return queue_bb(bbl, bb);
}
int backed_block_split(struct backed_block_list* bbl, struct backed_block* bb,
unsigned int max_len) {
struct backed_block* new_bb;
max_len = ALIGN_DOWN(max_len, bbl->block_size);
if (bb->len <= max_len) {
return 0;
}
new_bb = reinterpret_cast<backed_block*>(malloc(sizeof(struct backed_block)));
if (new_bb == nullptr) {
return -ENOMEM;
}
*new_bb = *bb;
new_bb->len = bb->len - max_len;
new_bb->block = bb->block + max_len / bbl->block_size;
new_bb->next = bb->next;
bb->next = new_bb;
bb->len = max_len;
switch (bb->type) {
case BACKED_BLOCK_DATA:
new_bb->data.data = (char*)bb->data.data + max_len;
break;
case BACKED_BLOCK_FILE:
new_bb->file.offset += max_len;
break;
case BACKED_BLOCK_FD:
new_bb->fd.offset += max_len;
break;
case BACKED_BLOCK_FILL:
break;
}
return 0;
}

58
backed_block.h
View File

@ -23,42 +23,40 @@ struct backed_block_list;
struct backed_block;
enum backed_block_type {
BACKED_BLOCK_DATA,
BACKED_BLOCK_FILE,
BACKED_BLOCK_FD,
BACKED_BLOCK_FILL,
BACKED_BLOCK_DATA,
BACKED_BLOCK_FILE,
BACKED_BLOCK_FD,
BACKED_BLOCK_FILL,
};
int backed_block_add_data(struct backed_block_list *bbl, void *data,
int backed_block_add_data(struct backed_block_list* bbl, void* data, unsigned int len,
unsigned int block);
int backed_block_add_fill(struct backed_block_list* bbl, unsigned int fill_val, unsigned int len,
unsigned int block);
int backed_block_add_file(struct backed_block_list* bbl, const char* filename, int64_t offset,
unsigned int len, unsigned int block);
int backed_block_add_fill(struct backed_block_list *bbl, unsigned int fill_val,
unsigned int len, unsigned int block);
int backed_block_add_file(struct backed_block_list *bbl, const char *filename,
int64_t offset, unsigned int len, unsigned int block);
int backed_block_add_fd(struct backed_block_list *bbl, int fd,
int64_t offset, unsigned int len, unsigned int block);
int backed_block_add_fd(struct backed_block_list* bbl, int fd, int64_t offset, unsigned int len,
unsigned int block);
struct backed_block *backed_block_iter_new(struct backed_block_list *bbl);
struct backed_block *backed_block_iter_next(struct backed_block *bb);
unsigned int backed_block_len(struct backed_block *bb);
unsigned int backed_block_block(struct backed_block *bb);
void *backed_block_data(struct backed_block *bb);
const char *backed_block_filename(struct backed_block *bb);
int backed_block_fd(struct backed_block *bb);
int64_t backed_block_file_offset(struct backed_block *bb);
uint32_t backed_block_fill_val(struct backed_block *bb);
enum backed_block_type backed_block_type(struct backed_block *bb);
int backed_block_split(struct backed_block_list *bbl, struct backed_block *bb,
unsigned int max_len);
struct backed_block* backed_block_iter_new(struct backed_block_list* bbl);
struct backed_block* backed_block_iter_next(struct backed_block* bb);
unsigned int backed_block_len(struct backed_block* bb);
unsigned int backed_block_block(struct backed_block* bb);
void* backed_block_data(struct backed_block* bb);
const char* backed_block_filename(struct backed_block* bb);
int backed_block_fd(struct backed_block* bb);
int64_t backed_block_file_offset(struct backed_block* bb);
uint32_t backed_block_fill_val(struct backed_block* bb);
enum backed_block_type backed_block_type(struct backed_block* bb);
int backed_block_split(struct backed_block_list* bbl, struct backed_block* bb, unsigned int max_len);
struct backed_block *backed_block_iter_new(struct backed_block_list *bbl);
struct backed_block *backed_block_iter_next(struct backed_block *bb);
struct backed_block* backed_block_iter_new(struct backed_block_list* bbl);
struct backed_block* backed_block_iter_next(struct backed_block* bb);
struct backed_block_list *backed_block_list_new(unsigned int block_size);
void backed_block_list_destroy(struct backed_block_list *bbl);
struct backed_block_list* backed_block_list_new(unsigned int block_size);
void backed_block_list_destroy(struct backed_block_list* bbl);
void backed_block_list_move(struct backed_block_list *from,
struct backed_block_list *to, struct backed_block *start,
struct backed_block *end);
void backed_block_list_move(struct backed_block_list* from, struct backed_block_list* to,
struct backed_block* start, struct backed_block* end);
#endif

4
defs.h
View File

@ -17,7 +17,7 @@
#ifndef _LIBSPARSE_DEFS_H_
#ifndef __unused
#define __unused __attribute__((__unused__))
#define __unused __attribute__((__unused__))
#endif
#endif /* _LIBSPARSE_DEFS_H_ */
#endif /* _LIBSPARSE_DEFS_H_ */

115
img2simg.c
View File

@ -1,115 +0,0 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <sparse/sparse.h>
#ifndef O_BINARY
#define O_BINARY 0
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t
#endif
void usage()
{
fprintf(stderr, "Usage: img2simg <raw_image_file> <sparse_image_file> [<block_size>]\n");
}
int main(int argc, char *argv[])
{
int in;
int out;
int ret;
struct sparse_file *s;
unsigned int block_size = 4096;
off64_t len;
if (argc < 3 || argc > 4) {
usage();
exit(-1);
}
if (argc == 4) {
block_size = atoi(argv[3]);
}
if (block_size < 1024 || block_size % 4 != 0) {
usage();
exit(-1);
}
if (strcmp(argv[1], "-") == 0) {
in = STDIN_FILENO;
} else {
in = open(argv[1], O_RDONLY | O_BINARY);
if (in < 0) {
fprintf(stderr, "Cannot open input file %s\n", argv[1]);
exit(-1);
}
}
if (strcmp(argv[2], "-") == 0) {
out = STDOUT_FILENO;
} else {
out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
if (out < 0) {
fprintf(stderr, "Cannot open output file %s\n", argv[2]);
exit(-1);
}
}
len = lseek64(in, 0, SEEK_END);
lseek64(in, 0, SEEK_SET);
s = sparse_file_new(block_size, len);
if (!s) {
fprintf(stderr, "Failed to create sparse file\n");
exit(-1);
}
sparse_file_verbose(s);
ret = sparse_file_read(s, in, false, false);
if (ret) {
fprintf(stderr, "Failed to read file\n");
exit(-1);
}
ret = sparse_file_write(s, out, false, true, false);
if (ret) {
fprintf(stderr, "Failed to write sparse file\n");
exit(-1);
}
close(in);
close(out);
exit(0);
}

112
img2simg.cpp Normal file
View File

@ -0,0 +1,112 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <sparse/sparse.h>
#ifndef O_BINARY
#define O_BINARY 0
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t
#endif
void usage() {
fprintf(stderr, "Usage: img2simg <raw_image_file> <sparse_image_file> [<block_size>]\n");
}
int main(int argc, char* argv[]) {
int in;
int out;
int ret;
struct sparse_file* s;
unsigned int block_size = 4096;
off64_t len;
if (argc < 3 || argc > 4) {
usage();
exit(-1);
}
if (argc == 4) {
block_size = atoi(argv[3]);
}
if (block_size < 1024 || block_size % 4 != 0) {
usage();
exit(-1);
}
if (strcmp(argv[1], "-") == 0) {
in = STDIN_FILENO;
} else {
in = open(argv[1], O_RDONLY | O_BINARY);
if (in < 0) {
fprintf(stderr, "Cannot open input file %s\n", argv[1]);
exit(-1);
}
}
if (strcmp(argv[2], "-") == 0) {
out = STDOUT_FILENO;
} else {
out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
if (out < 0) {
fprintf(stderr, "Cannot open output file %s\n", argv[2]);
exit(-1);
}
}
len = lseek64(in, 0, SEEK_END);
lseek64(in, 0, SEEK_SET);
s = sparse_file_new(block_size, len);
if (!s) {
fprintf(stderr, "Failed to create sparse file\n");
exit(-1);
}
sparse_file_verbose(s);
ret = sparse_file_read(s, in, false, false);
if (ret) {
fprintf(stderr, "Failed to read file\n");
exit(-1);
}
ret = sparse_file_write(s, out, false, true, false);
if (ret) {
fprintf(stderr, "Failed to write sparse file\n");
exit(-1);
}
close(in);
close(out);
exit(0);
}

42
include/sparse/sparse.h
View File

@ -18,6 +18,7 @@
#define _LIBSPARSE_SPARSE_H_
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
@ -26,6 +27,11 @@ extern "C" {
struct sparse_file;
// The callbacks in sparse_file_callback() and sparse_file_foreach_chunk() take
// size_t as the length type (was `int` in past). This allows clients to keep
// their codes compatibile with both versions as needed.
#define SPARSE_CALLBACK_USES_SIZE_T
/**
* sparse_file_new - create a new sparse file cookie
*
@ -201,7 +207,7 @@ unsigned int sparse_file_block_size(struct sparse_file *s);
* Returns 0 on success, negative errno on error.
*/
int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc,
int (*write)(void *priv, const void *data, int len), void *priv);
int (*write)(void *priv, const void *data, size_t len), void *priv);
/**
* sparse_file_foreach_chunk - call a callback for data blocks in sparse file
@ -218,7 +224,7 @@ int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc,
* Returns 0 on success, negative errno on error.
*/
int sparse_file_foreach_chunk(struct sparse_file *s, bool sparse, bool crc,
int (*write)(void *priv, const void *data, int len, unsigned int block,
int (*write)(void *priv, const void *data, size_t len, unsigned int block,
unsigned int nr_blocks),
void *priv);
/**
@ -240,9 +246,24 @@ int sparse_file_foreach_chunk(struct sparse_file *s, bool sparse, bool crc,
int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc);
/**
* sparse_file_import - import an existing sparse file
* sparse_file_read_buf - read a buffer into a sparse file cookie
*
* @s - sparse file cookie
* @buf - buffer to read from
* @crc - verify the crc of a file in the Android sparse file format
*
* Reads a buffer into a sparse file cookie. The buffer must remain
* valid until the sparse file cookie is freed. If crc is true, the
* crc of the sparse file will be verified.
*
* Returns 0 on success, negative errno on error.
*/
int sparse_file_read_buf(struct sparse_file *s, char *buf, bool crc);
/**
* sparse_file_import - import an existing sparse file
*
* @fd - file descriptor to read from
* @verbose - print verbose errors while reading the sparse file
* @crc - verify the crc of a file in the Android sparse file format
*
@ -254,6 +275,21 @@ int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc);
*/
struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc);
/**
* sparse_file_import_buf - import an existing sparse file from a buffer
*
* @buf - buffer to read from
* @verbose - print verbose errors while reading the sparse file
* @crc - verify the crc of a file in the Android sparse file format
*
* Reads existing sparse file data into a sparse file cookie, recreating the same
* sparse cookie that was used to write it. If verbose is true, prints verbose
* errors when the sparse file is formatted incorrectly.
*
* Returns a new sparse file cookie on success, NULL on error.
*/
struct sparse_file *sparse_file_import_buf(char* buf, bool verbose, bool crc);
/**
* sparse_file_import_auto - import an existing sparse or normal file
*

769
output_file.c
View File

@ -1,769 +0,0 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <fcntl.h>
#include <inttypes.h>
#include <limits.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <zlib.h>
#include "defs.h"
#include "output_file.h"
#include "sparse_crc32.h"
#include "sparse_format.h"
#ifndef USE_MINGW
#include <sys/mman.h>
#define O_BINARY 0
#else
#define ftruncate64 ftruncate
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define ftruncate64 ftruncate
#define mmap64 mmap
#define off64_t off_t
#endif
#define min(a, b) \
({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
#define SPARSE_HEADER_MAJOR_VER 1
#define SPARSE_HEADER_MINOR_VER 0
#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
#define container_of(inner, outer_t, elem) \
((outer_t *)((char *)(inner) - offsetof(outer_t, elem)))
struct output_file_ops {
int (*open) (struct output_file *, int fd);
int (*skip) (struct output_file *, int64_t);
int (*pad) (struct output_file *, int64_t);
int (*write) (struct output_file *, void *, size_t);
void (*close) (struct output_file *);
};
struct sparse_file_ops {
int (*write_data_chunk) (struct output_file * out, unsigned int len, void *data);
int (*write_fill_chunk) (struct output_file * out, unsigned int len, uint32_t fill_val);
int (*write_skip_chunk) (struct output_file * out, int64_t len);
int (*write_end_chunk) (struct output_file * out);
};
struct output_file {
int64_t cur_out_ptr;
unsigned int chunk_cnt;
uint32_t crc32;
struct output_file_ops *ops;
struct sparse_file_ops *sparse_ops;
int use_crc;
unsigned int block_size;
int64_t len;
char *zero_buf;
uint32_t *fill_buf;
char *buf;
};
struct output_file_gz {
struct output_file out;
gzFile gz_fd;
};
#define to_output_file_gz(_o) \
container_of((_o), struct output_file_gz, out)
struct output_file_normal {
struct output_file out;
int fd;
};
#define to_output_file_normal(_o) \
container_of((_o), struct output_file_normal, out)
struct output_file_callback {
struct output_file out;
void *priv;
int (*write) (void *priv, const void *buf, int len);
};
#define to_output_file_callback(_o) \
container_of((_o), struct output_file_callback, out)
static int file_open(struct output_file *out, int fd)
{
struct output_file_normal *outn = to_output_file_normal(out);
outn->fd = fd;
return 0;
}
static int file_skip(struct output_file *out, int64_t cnt)
{
off64_t ret;
struct output_file_normal *outn = to_output_file_normal(out);
ret = lseek64(outn->fd, cnt, SEEK_CUR);
if (ret < 0) {
error_errno("lseek64");
return -1;
}
return 0;
}
static int file_pad(struct output_file *out, int64_t len)
{
int ret;
struct output_file_normal *outn = to_output_file_normal(out);
ret = ftruncate64(outn->fd, len);
if (ret < 0) {
return -errno;
}
return 0;
}
static int file_write(struct output_file *out, void *data, size_t len)
{
ssize_t ret;
struct output_file_normal *outn = to_output_file_normal(out);
while (len > 0) {
ret = write(outn->fd, data, len);
if (ret < 0) {
if (errno == EINTR) {
continue;
}
error_errno("write");
return -1;
}
data = (char *)data + ret;
len -= ret;
}
return 0;
}
static void file_close(struct output_file *out)
{
struct output_file_normal *outn = to_output_file_normal(out);
free(outn);
}
static struct output_file_ops file_ops = {
.open = file_open,
.skip = file_skip,
.pad = file_pad,
.write = file_write,
.close = file_close,
};
static int gz_file_open(struct output_file *out, int fd)
{
struct output_file_gz *outgz = to_output_file_gz(out);
outgz->gz_fd = gzdopen(fd, "wb9");
if (!outgz->gz_fd) {
error_errno("gzopen");
return -errno;
}
return 0;
}
static int gz_file_skip(struct output_file *out, int64_t cnt)
{
off64_t ret;
struct output_file_gz *outgz = to_output_file_gz(out);
ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
if (ret < 0) {
error_errno("gzseek");
return -1;
}
return 0;
}
static int gz_file_pad(struct output_file *out, int64_t len)
{
off64_t ret;
struct output_file_gz *outgz = to_output_file_gz(out);
ret = gztell(outgz->gz_fd);
if (ret < 0) {
return -1;
}
if (ret >= len) {
return 0;
}
ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
if (ret < 0) {
return -1;
}
gzwrite(outgz->gz_fd, "", 1);
return 0;
}
static int gz_file_write(struct output_file *out, void *data, size_t len)
{
int ret;
struct output_file_gz *outgz = to_output_file_gz(out);
while (len > 0) {
ret = gzwrite(outgz->gz_fd, data,
min(len, (unsigned int)INT_MAX));
if (ret == 0) {
error("gzwrite %s", gzerror(outgz->gz_fd, NULL));
return -1;
}
len -= ret;
data = (char *)data + ret;
}
return 0;
}
static void gz_file_close(struct output_file *out)
{
struct output_file_gz *outgz = to_output_file_gz(out);
gzclose(outgz->gz_fd);
free(outgz);
}
static struct output_file_ops gz_file_ops = {
.open = gz_file_open,
.skip = gz_file_skip,
.pad = gz_file_pad,
.write = gz_file_write,
.close = gz_file_close,
};
static int callback_file_open(struct output_file *out __unused, int fd __unused)
{
return 0;
}
static int callback_file_skip(struct output_file *out, int64_t off)
{
struct output_file_callback *outc = to_output_file_callback(out);
int to_write;
int ret;
while (off > 0) {
to_write = min(off, (int64_t) INT_MAX);
ret = outc->write(outc->priv, NULL, to_write);
if (ret < 0) {
return ret;
}
off -= to_write;
}
return 0;
}
static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
{
return -1;
}
static int callback_file_write(struct output_file *out, void *data, size_t len)
{
struct output_file_callback *outc = to_output_file_callback(out);
return outc->write(outc->priv, data, len);
}
static void callback_file_close(struct output_file *out)
{
struct output_file_callback *outc = to_output_file_callback(out);
free(outc);
}
static struct output_file_ops callback_file_ops = {
.open = callback_file_open,
.skip = callback_file_skip,
.pad = callback_file_pad,
.write = callback_file_write,
.close = callback_file_close,
};
int read_all(int fd, void *buf, size_t len)
{
size_t total = 0;
int ret;
char *ptr = buf;
while (total < len) {
ret = read(fd, ptr, len - total);
if (ret < 0)
return -errno;
if (ret == 0)
return -EINVAL;
ptr += ret;
total += ret;
}
return 0;
}
static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
{
chunk_header_t chunk_header;
int ret;
if (skip_len % out->block_size) {
error("don't care size %" PRIi64 " is not a multiple of the block size %u",
skip_len, out->block_size);
return -1;
}
/* We are skipping data, so emit a don't care chunk. */
chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = skip_len / out->block_size;
chunk_header.total_sz = CHUNK_HEADER_LEN;
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0)
return -1;
out->cur_out_ptr += skip_len;
out->chunk_cnt++;
return 0;
}
static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val)
{
chunk_header_t chunk_header;
int rnd_up_len, count;
int ret;
/* Round up the fill length to a multiple of the block size */
rnd_up_len = ALIGN(len, out->block_size);
/* Finally we can safely emit a chunk of data */
chunk_header.chunk_type = CHUNK_TYPE_FILL;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = rnd_up_len / out->block_size;
chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0)
return -1;
ret = out->ops->write(out, &fill_val, sizeof(fill_val));
if (ret < 0)
return -1;
if (out->use_crc) {
count = out->block_size / sizeof(uint32_t);
while (count--)
out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
}
out->cur_out_ptr += rnd_up_len;
out->chunk_cnt++;
return 0;
}
static int write_sparse_data_chunk(struct output_file *out, unsigned int len, void *data)
{
chunk_header_t chunk_header;
int rnd_up_len, zero_len;
int ret;
/* Round up the data length to a multiple of the block size */
rnd_up_len = ALIGN(len, out->block_size);
zero_len = rnd_up_len - len;
/* Finally we can safely emit a chunk of data */
chunk_header.chunk_type = CHUNK_TYPE_RAW;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = rnd_up_len / out->block_size;
chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0)
return -1;
ret = out->ops->write(out, data, len);
if (ret < 0)
return -1;
if (zero_len) {
ret = out->ops->write(out, out->zero_buf, zero_len);
if (ret < 0)
return -1;
}
if (out->use_crc) {
out->crc32 = sparse_crc32(out->crc32, data, len);
if (zero_len)
out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
}
out->cur_out_ptr += rnd_up_len;
out->chunk_cnt++;
return 0;
}
int write_sparse_end_chunk(struct output_file *out)
{
chunk_header_t chunk_header;
int ret;
if (out->use_crc) {
chunk_header.chunk_type = CHUNK_TYPE_CRC32;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = 0;
chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0) {
return ret;
}
out->ops->write(out, &out->crc32, 4);
if (ret < 0) {
return ret;
}
out->chunk_cnt++;
}
return 0;
}
static struct sparse_file_ops sparse_file_ops = {
.write_data_chunk = write_sparse_data_chunk,
.write_fill_chunk = write_sparse_fill_chunk,
.write_skip_chunk = write_sparse_skip_chunk,
.write_end_chunk = write_sparse_end_chunk,
};
static int write_normal_data_chunk(struct output_file *out, unsigned int len, void *data)
{
int ret;
unsigned int rnd_up_len = ALIGN(len, out->block_size);
ret = out->ops->write(out, data, len);
if (ret < 0) {
return ret;
}
if (rnd_up_len > len) {
ret = out->ops->skip(out, rnd_up_len - len);
}
return ret;
}
static int write_normal_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val)
{
int ret;
unsigned int i;
unsigned int write_len;
/* Initialize fill_buf with the fill_val */
for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
out->fill_buf[i] = fill_val;
}
while (len) {
write_len = min(len, out->block_size);
ret = out->ops->write(out, out->fill_buf, write_len);
if (ret < 0) {
return ret;
}
len -= write_len;
}
return 0;
}
static int write_normal_skip_chunk(struct output_file *out, int64_t len)
{
return out->ops->skip(out, len);
}
int write_normal_end_chunk(struct output_file *out)
{
return out->ops->pad(out, out->len);
}
static struct sparse_file_ops normal_file_ops = {
.write_data_chunk = write_normal_data_chunk,
.write_fill_chunk = write_normal_fill_chunk,
.write_skip_chunk = write_normal_skip_chunk,
.write_end_chunk = write_normal_end_chunk,
};
void output_file_close(struct output_file *out)
{
out->sparse_ops->write_end_chunk(out);
out->ops->close(out);
}
static int output_file_init(struct output_file *out, int block_size,
int64_t len, bool sparse, int chunks, bool crc)
{
int ret;
out->len = len;
out->block_size = block_size;
out->cur_out_ptr = 0ll;
out->chunk_cnt = 0;
out->crc32 = 0;
out->use_crc = crc;
out->zero_buf = calloc(block_size, 1);
if (!out->zero_buf) {
error_errno("malloc zero_buf");
return -ENOMEM;
}
out->fill_buf = calloc(block_size, 1);
if (!out->fill_buf) {
error_errno("malloc fill_buf");
ret = -ENOMEM;
goto err_fill_buf;
}
if (sparse) {
out->sparse_ops = &sparse_file_ops;
} else {
out->sparse_ops = &normal_file_ops;
}
if (sparse) {
sparse_header_t sparse_header = {
.magic = SPARSE_HEADER_MAGIC,
.major_version = SPARSE_HEADER_MAJOR_VER,
.minor_version = SPARSE_HEADER_MINOR_VER,
.file_hdr_sz = SPARSE_HEADER_LEN,
.chunk_hdr_sz = CHUNK_HEADER_LEN,
.blk_sz = out->block_size,
.total_blks = out->len / out->block_size,
.total_chunks = chunks,
.image_checksum = 0
};
if (out->use_crc) {
sparse_header.total_chunks++;
}
ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
if (ret < 0) {
goto err_write;
}
}
return 0;
err_write:
free(out->fill_buf);
err_fill_buf:
free(out->zero_buf);
return ret;
}
static struct output_file *output_file_new_gz(void)
{
struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
if (!outgz) {
error_errno("malloc struct outgz");
return NULL;
}
outgz->out.ops = &gz_file_ops;
return &outgz->out;
}
static struct output_file *output_file_new_normal(void)
{
struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
if (!outn) {
error_errno("malloc struct outn");
return NULL;
}
outn->out.ops = &file_ops;
return &outn->out;
}
struct output_file *output_file_open_callback(int (*write) (void *, const void *, int),
void *priv, unsigned int block_size, int64_t len,
int gz __unused, int sparse, int chunks, int crc)
{
int ret;
struct output_file_callback *outc;
outc = calloc(1, sizeof(struct output_file_callback));
if (!outc) {
error_errno("malloc struct outc");
return NULL;
}
outc->out.ops = &callback_file_ops;
outc->priv = priv;
outc->write = write;
ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
if (ret < 0) {
free(outc);
return NULL;
}
return &outc->out;
}
struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
int gz, int sparse, int chunks, int crc)
{
int ret;
struct output_file *out;
if (gz) {
out = output_file_new_gz();
} else {
out = output_file_new_normal();
}
if (!out) {
return NULL;
}
out->ops->open(out, fd);
ret = output_file_init(out, block_size, len, sparse, chunks, crc);
if (ret < 0) {
free(out);
return NULL;
}
return out;
}
/* Write a contiguous region of data blocks from a memory buffer */
int write_data_chunk(struct output_file *out, unsigned int len, void *data)
{
return out->sparse_ops->write_data_chunk(out, len, data);
}
/* Write a contiguous region of data blocks with a fill value */
int write_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val)
{
return out->sparse_ops->write_fill_chunk(out, len, fill_val);
}
int write_fd_chunk(struct output_file *out, unsigned int len, int fd, int64_t offset)
{
int ret;
int64_t aligned_offset;
int aligned_diff;
uint64_t buffer_size;
char *ptr;
#ifdef _SC_PAGESIZE
aligned_offset = offset & ~(sysconf(_SC_PAGESIZE) - 1);
#else
aligned_offset = offset & ~(4096 - 1);
#endif
aligned_diff = offset - aligned_offset;
buffer_size = (uint64_t)len + (uint64_t)aligned_diff;
#ifndef USE_MINGW
if (buffer_size > SIZE_MAX)
return -E2BIG;
char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
aligned_offset);
if (data == MAP_FAILED) {
return -errno;
}
ptr = data + aligned_diff;
#else
off64_t pos;
char *data = malloc(len);
if (!data) {
return -errno;
}
pos = lseek64(fd, offset, SEEK_SET);
if (pos < 0) {
free(data);
return -errno;
}
ret = read_all(fd, data, len);
if (ret < 0) {
free(data);
return ret;
}
ptr = data;
#endif
ret = out->sparse_ops->write_data_chunk(out, len, ptr);
#ifndef USE_MINGW
munmap(data, buffer_size);
#else
free(data);
#endif
return ret;
}
/* Write a contiguous region of data blocks from a file */
int write_file_chunk(struct output_file *out, unsigned int len, const char *file, int64_t offset)
{
int ret;
int file_fd = open(file, O_RDONLY | O_BINARY);
if (file_fd < 0) {
return -errno;
}
ret = write_fd_chunk(out, len, file_fd, offset);
close(file_fd);
return ret;
}
int write_skip_chunk(struct output_file *out, int64_t len)
{
return out->sparse_ops->write_skip_chunk(out, len);
}

720
output_file.cpp Normal file
View File

@ -0,0 +1,720 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <fcntl.h>
#include <inttypes.h>
#include <limits.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <zlib.h>
#include "defs.h"
#include "output_file.h"
#include "sparse_crc32.h"
#include "sparse_format.h"
#ifndef _WIN32
#include <sys/mman.h>
#define O_BINARY 0
#else
#define ftruncate64 ftruncate
#endif
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define ftruncate64 ftruncate
#define mmap64 mmap
#define off64_t off_t
#endif
#define min(a, b) \
({ \
typeof(a) _a = (a); \
typeof(b) _b = (b); \
(_a < _b) ? _a : _b; \
})
#define SPARSE_HEADER_MAJOR_VER 1
#define SPARSE_HEADER_MINOR_VER 0
#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
#define container_of(inner, outer_t, elem) ((outer_t*)((char*)(inner)-offsetof(outer_t, elem)))
struct output_file_ops {
int (*open)(struct output_file*, int fd);
int (*skip)(struct output_file*, int64_t);
int (*pad)(struct output_file*, int64_t);
int (*write)(struct output_file*, void*, size_t);
void (*close)(struct output_file*);
};
struct sparse_file_ops {
int (*write_data_chunk)(struct output_file* out, unsigned int len, void* data);
int (*write_fill_chunk)(struct output_file* out, unsigned int len, uint32_t fill_val);
int (*write_skip_chunk)(struct output_file* out, int64_t len);
int (*write_end_chunk)(struct output_file* out);
};
struct output_file {
int64_t cur_out_ptr;
unsigned int chunk_cnt;
uint32_t crc32;
struct output_file_ops* ops;
struct sparse_file_ops* sparse_ops;
int use_crc;
unsigned int block_size;
int64_t len;
char* zero_buf;
uint32_t* fill_buf;
char* buf;
};
struct output_file_gz {
struct output_file out;
gzFile gz_fd;
};
#define to_output_file_gz(_o) container_of((_o), struct output_file_gz, out)
struct output_file_normal {
struct output_file out;
int fd;
};
#define to_output_file_normal(_o) container_of((_o), struct output_file_normal, out)
struct output_file_callback {
struct output_file out;
void* priv;
int (*write)(void* priv, const void* buf, size_t len);
};
#define to_output_file_callback(_o) container_of((_o), struct output_file_callback, out)
static int file_open(struct output_file* out, int fd) {
struct output_file_normal* outn = to_output_file_normal(out);
outn->fd = fd;
return 0;
}
static int file_skip(struct output_file* out, int64_t cnt) {
off64_t ret;
struct output_file_normal* outn = to_output_file_normal(out);
ret = lseek64(outn->fd, cnt, SEEK_CUR);
if (ret < 0) {
error_errno("lseek64");
return -1;
}
return 0;
}
static int file_pad(struct output_file* out, int64_t len) {
int ret;
struct output_file_normal* outn = to_output_file_normal(out);
ret = ftruncate64(outn->fd, len);
if (ret < 0) {
return -errno;
}
return 0;
}
static int file_write(struct output_file* out, void* data, size_t len) {
ssize_t ret;
struct output_file_normal* outn = to_output_file_normal(out);
while (len > 0) {
ret = write(outn->fd, data, len);
if (ret < 0) {
if (errno == EINTR) {
continue;
}
error_errno("write");
return -1;
}
data = (char*)data + ret;
len -= ret;
}
return 0;
}
static void file_close(struct output_file* out) {
struct output_file_normal* outn = to_output_file_normal(out);
free(outn);
}
static struct output_file_ops file_ops = {
.open = file_open,
.skip = file_skip,
.pad = file_pad,
.write = file_write,
.close = file_close,
};
static int gz_file_open(struct output_file* out, int fd) {
struct output_file_gz* outgz = to_output_file_gz(out);
outgz->gz_fd = gzdopen(fd, "wb9");
if (!outgz->gz_fd) {
error_errno("gzopen");
return -errno;
}
return 0;
}
static int gz_file_skip(struct output_file* out, int64_t cnt) {
off64_t ret;
struct output_file_gz* outgz = to_output_file_gz(out);
ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
if (ret < 0) {
error_errno("gzseek");
return -1;
}
return 0;
}
static int gz_file_pad(struct output_file* out, int64_t len) {
off64_t ret;
struct output_file_gz* outgz = to_output_file_gz(out);
ret = gztell(outgz->gz_fd);
if (ret < 0) {
return -1;
}
if (ret >= len) {
return 0;
}
ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
if (ret < 0) {
return -1;
}
gzwrite(outgz->gz_fd, "", 1);
return 0;
}
static int gz_file_write(struct output_file* out, void* data, size_t len) {
int ret;
struct output_file_gz* outgz = to_output_file_gz(out);
while (len > 0) {
ret = gzwrite(outgz->gz_fd, data, min(len, (unsigned int)INT_MAX));
if (ret == 0) {
error("gzwrite %s", gzerror(outgz->gz_fd, nullptr));
return -1;
}
len -= ret;
data = (char*)data + ret;
}
return 0;
}
static void gz_file_close(struct output_file* out) {
struct output_file_gz* outgz = to_output_file_gz(out);
gzclose(outgz->gz_fd);
free(outgz);
}
static struct output_file_ops gz_file_ops = {
.open = gz_file_open,
.skip = gz_file_skip,
.pad = gz_file_pad,
.write = gz_file_write,
.close = gz_file_close,
};
static int callback_file_open(struct output_file* out __unused, int fd __unused) {
return 0;
}
static int callback_file_skip(struct output_file* out, int64_t off) {
struct output_file_callback* outc = to_output_file_callback(out);
int to_write;
int ret;
while (off > 0) {
to_write = min(off, (int64_t)INT_MAX);
ret = outc->write(outc->priv, nullptr, to_write);
if (ret < 0) {
return ret;
}
off -= to_write;
}
return 0;
}
static int callback_file_pad(struct output_file* out __unused, int64_t len __unused) {
return -1;
}
static int callback_file_write(struct output_file* out, void* data, size_t len) {
struct output_file_callback* outc = to_output_file_callback(out);
return outc->write(outc->priv, data, len);
}
static void callback_file_close(struct output_file* out) {
struct output_file_callback* outc = to_output_file_callback(out);
free(outc);
}
static struct output_file_ops callback_file_ops = {
.open = callback_file_open,
.skip = callback_file_skip,
.pad = callback_file_pad,
.write = callback_file_write,
.close = callback_file_close,
};
int read_all(int fd, void* buf, size_t len) {
size_t total = 0;
int ret;
char* ptr = reinterpret_cast<char*>(buf);
while (total < len) {
ret = read(fd, ptr, len - total);
if (ret < 0) return -errno;
if (ret == 0) return -EINVAL;
ptr += ret;
total += ret;
}
return 0;
}
static int write_sparse_skip_chunk(struct output_file* out, int64_t skip_len) {
chunk_header_t chunk_header;
int ret;
if (skip_len % out->block_size) {
error("don't care size %" PRIi64 " is not a multiple of the block size %u", skip_len,
out->block_size);
return -1;
}
/* We are skipping data, so emit a don't care chunk. */
chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = skip_len / out->block_size;
chunk_header.total_sz = CHUNK_HEADER_LEN;
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0) return -1;
out->cur_out_ptr += skip_len;
out->chunk_cnt++;
return 0;
}
static int write_sparse_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) {
chunk_header_t chunk_header;
int rnd_up_len, count;
int ret;
/* Round up the fill length to a multiple of the block size */
rnd_up_len = ALIGN(len, out->block_size);
/* Finally we can safely emit a chunk of data */
chunk_header.chunk_type = CHUNK_TYPE_FILL;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = rnd_up_len / out->block_size;
chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0) return -1;
ret = out->ops->write(out, &fill_val, sizeof(fill_val));
if (ret < 0) return -1;
if (out->use_crc) {
count = out->block_size / sizeof(uint32_t);
while (count--) out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
}
out->cur_out_ptr += rnd_up_len;
out->chunk_cnt++;
return 0;
}
static int write_sparse_data_chunk(struct output_file* out, unsigned int len, void* data) {
chunk_header_t chunk_header;
int rnd_up_len, zero_len;
int ret;
/* Round up the data length to a multiple of the block size */
rnd_up_len = ALIGN(len, out->block_size);
zero_len = rnd_up_len - len;
/* Finally we can safely emit a chunk of data */
chunk_header.chunk_type = CHUNK_TYPE_RAW;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = rnd_up_len / out->block_size;
chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0) return -1;
ret = out->ops->write(out, data, len);
if (ret < 0) return -1;
if (zero_len) {
ret = out->ops->write(out, out->zero_buf, zero_len);
if (ret < 0) return -1;
}
if (out->use_crc) {
out->crc32 = sparse_crc32(out->crc32, data, len);
if (zero_len) out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
}
out->cur_out_ptr += rnd_up_len;
out->chunk_cnt++;
return 0;
}
int write_sparse_end_chunk(struct output_file* out) {
chunk_header_t chunk_header;
int ret;
if (out->use_crc) {
chunk_header.chunk_type = CHUNK_TYPE_CRC32;
chunk_header.reserved1 = 0;
chunk_header.chunk_sz = 0;
chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
if (ret < 0) {
return ret;
}
out->ops->write(out, &out->crc32, 4);
if (ret < 0) {
return ret;
}
out->chunk_cnt++;
}
return 0;
}
static struct sparse_file_ops sparse_file_ops = {
.write_data_chunk = write_sparse_data_chunk,
.write_fill_chunk = write_sparse_fill_chunk,
.write_skip_chunk = write_sparse_skip_chunk,
.write_end_chunk = write_sparse_end_chunk,
};
static int write_normal_data_chunk(struct output_file* out, unsigned int len, void* data) {
int ret;
unsigned int rnd_up_len = ALIGN(len, out->block_size);
ret = out->ops->write(out, data, len);
if (ret < 0) {
return ret;
}
if (rnd_up_len > len) {
ret = out->ops->skip(out, rnd_up_len - len);
}
return ret;
}
static int write_normal_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) {
int ret;
unsigned int i;
unsigned int write_len;
/* Initialize fill_buf with the fill_val */
for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
out->fill_buf[i] = fill_val;
}
while (len) {
write_len = min(len, out->block_size);
ret = out->ops->write(out, out->fill_buf, write_len);
if (ret < 0) {
return ret;
}
len -= write_len;
}
return 0;
}
static int write_normal_skip_chunk(struct output_file* out, int64_t len) {
return out->ops->skip(out, len);
}
int write_normal_end_chunk(struct output_file* out) {
return out->ops->pad(out, out->len);
}
static struct sparse_file_ops normal_file_ops = {
.write_data_chunk = write_normal_data_chunk,
.write_fill_chunk = write_normal_fill_chunk,
.write_skip_chunk = write_normal_skip_chunk,
.write_end_chunk = write_normal_end_chunk,
};
void output_file_close(struct output_file* out) {
out->sparse_ops->write_end_chunk(out);
out->ops->close(out);
}
static int output_file_init(struct output_file* out, int block_size, int64_t len, bool sparse,
int chunks, bool crc) {
int ret;
out->len = len;
out->block_size = block_size;
out->cur_out_ptr = 0LL;
out->chunk_cnt = 0;
out->crc32 = 0;
out->use_crc = crc;
out->zero_buf = reinterpret_cast<char*>(calloc(block_size, 1));
if (!out->zero_buf) {
error_errno("malloc zero_buf");
return -ENOMEM;
}
out->fill_buf = reinterpret_cast<uint32_t*>(calloc(block_size, 1));
if (!out->fill_buf) {
error_errno("malloc fill_buf");
ret = -ENOMEM;
goto err_fill_buf;
}
if (sparse) {
out->sparse_ops = &sparse_file_ops;
} else {
out->sparse_ops = &normal_file_ops;
}
if (sparse) {
sparse_header_t sparse_header = {
.magic = SPARSE_HEADER_MAGIC,
.major_version = SPARSE_HEADER_MAJOR_VER,
.minor_version = SPARSE_HEADER_MINOR_VER,
.file_hdr_sz = SPARSE_HEADER_LEN,
.chunk_hdr_sz = CHUNK_HEADER_LEN,
.blk_sz = out->block_size,
.total_blks = static_cast<unsigned>(DIV_ROUND_UP(out->len, out->block_size)),
.total_chunks = static_cast<unsigned>(chunks),
.image_checksum = 0};
if (out->use_crc) {
sparse_header.total_chunks++;
}
ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
if (ret < 0) {
goto err_write;
}
}
return 0;
err_write:
free(out->fill_buf);
err_fill_buf:
free(out->zero_buf);
return ret;
}
static struct output_file* output_file_new_gz(void) {
struct output_file_gz* outgz =
reinterpret_cast<struct output_file_gz*>(calloc(1, sizeof(struct output_file_gz)));
if (!outgz) {
error_errno("malloc struct outgz");
return nullptr;
}
outgz->out.ops = &gz_file_ops;
return &outgz->out;
}
static struct output_file* output_file_new_normal(void) {
struct output_file_normal* outn =
reinterpret_cast<struct output_file_normal*>(calloc(1, sizeof(struct output_file_normal)));
if (!outn) {
error_errno("malloc struct outn");
return nullptr;
}
outn->out.ops = &file_ops;
return &outn->out;
}
struct output_file* output_file_open_callback(int (*write)(void*, const void*, size_t), void* priv,
unsigned int block_size, int64_t len, int gz __unused,
int sparse, int chunks, int crc) {
int ret;
struct output_file_callback* outc;
outc =
reinterpret_cast<struct output_file_callback*>(calloc(1, sizeof(struct output_file_callback)));
if (!outc) {
error_errno("malloc struct outc");
return nullptr;
}
outc->out.ops = &callback_file_ops;
outc->priv = priv;
outc->write = write;
ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
if (ret < 0) {
free(outc);
return nullptr;
}
return &outc->out;
}
struct output_file* output_file_open_fd(int fd, unsigned int block_size, int64_t len, int gz,
int sparse, int chunks, int crc) {
int ret;
struct output_file* out;
if (gz) {
out = output_file_new_gz();
} else {
out = output_file_new_normal();
}
if (!out) {
return nullptr;
}
out->ops->open(out, fd);
ret = output_file_init(out, block_size, len, sparse, chunks, crc);
if (ret < 0) {
free(out);
return nullptr;
}
return out;
}
/* Write a contiguous region of data blocks from a memory buffer */
int write_data_chunk(struct output_file* out, unsigned int len, void* data) {
return out->sparse_ops->write_data_chunk(out, len, data);
}
/* Write a contiguous region of data blocks with a fill value */
int write_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) {
return out->sparse_ops->write_fill_chunk(out, len, fill_val);
}
int write_fd_chunk(struct output_file* out, unsigned int len, int fd, int64_t offset) {
int ret;
int64_t aligned_offset;
int aligned_diff;
uint64_t buffer_size;
char* ptr;
aligned_offset = offset & ~(4096 - 1);
aligned_diff = offset - aligned_offset;
buffer_size = (uint64_t)len + (uint64_t)aligned_diff;
#ifndef _WIN32
if (buffer_size > SIZE_MAX) return -E2BIG;
char* data =
reinterpret_cast<char*>(mmap64(nullptr, buffer_size, PROT_READ, MAP_SHARED, fd, aligned_offset));
if (data == MAP_FAILED) {
return -errno;
}
ptr = data + aligned_diff;
#else
off64_t pos;
char* data = reinterpret_cast<char*>(malloc(len));
if (!data) {
return -errno;
}
pos = lseek64(fd, offset, SEEK_SET);
if (pos < 0) {
free(data);
return -errno;
}
ret = read_all(fd, data, len);
if (ret < 0) {
free(data);
return ret;
}
ptr = data;
#endif
ret = out->sparse_ops->write_data_chunk(out, len, ptr);
#ifndef _WIN32
munmap(data, buffer_size);
#else
free(data);
#endif
return ret;
}
/* Write a contiguous region of data blocks from a file */
int write_file_chunk(struct output_file* out, unsigned int len, const char* file, int64_t offset) {
int ret;
int file_fd = open(file, O_RDONLY | O_BINARY);
if (file_fd < 0) {
return -errno;
}
ret = write_fd_chunk(out, len, file_fd, offset);
close(file_fd);
return ret;
}
int write_skip_chunk(struct output_file* out, int64_t len) {
return out->sparse_ops->write_skip_chunk(out, len);
}

32
output_file.h
View File

@ -17,22 +17,30 @@
#ifndef _OUTPUT_FILE_H_
#define _OUTPUT_FILE_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <sparse/sparse.h>
struct output_file;
struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
int gz, int sparse, int chunks, int crc);
struct output_file *output_file_open_callback(int (*write) (void *, const void *, int),
void *priv, unsigned int block_size, int64_t len,
int gz, int sparse, int chunks, int crc);
int write_data_chunk(struct output_file *out, unsigned int len, void *data);
int write_fill_chunk(struct output_file *out, unsigned int len, uint32_t fill_val);
int write_file_chunk(struct output_file *out, unsigned int len, const char *file, int64_t offset);
int write_fd_chunk(struct output_file *out, unsigned int len, int fd, int64_t offset);
int write_skip_chunk(struct output_file *out, int64_t len);
void output_file_close(struct output_file *out);
struct output_file* output_file_open_fd(int fd, unsigned int block_size, int64_t len, int gz,
int sparse, int chunks, int crc);
struct output_file* output_file_open_callback(int (*write)(void*, const void*, size_t), void* priv,
unsigned int block_size, int64_t len, int gz,
int sparse, int chunks, int crc);
int write_data_chunk(struct output_file* out, unsigned int len, void* data);
int write_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val);
int write_file_chunk(struct output_file* out, unsigned int len, const char* file, int64_t offset);
int write_fd_chunk(struct output_file* out, unsigned int len, int fd, int64_t offset);
int write_skip_chunk(struct output_file* out, int64_t len);
void output_file_close(struct output_file* out);
int read_all(int fd, void *buf, size_t len);
int read_all(int fd, void* buf, size_t len);
#ifdef __cplusplus
}
#endif
#endif

89
simg2img.c
View File

@ -1,89 +0,0 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <sparse/sparse.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#ifndef O_BINARY
#define O_BINARY 0
#endif
void usage()
{
fprintf(stderr, "Usage: simg2img <sparse_image_files> <raw_image_file>\n");
}
int main(int argc, char *argv[])
{
int in;
int out;
int i;
struct sparse_file *s;
if (argc < 3) {
usage();
exit(-1);
}
out = open(argv[argc - 1], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
if (out < 0) {
fprintf(stderr, "Cannot open output file %s\n", argv[argc - 1]);
exit(-1);
}
for (i = 1; i < argc - 1; i++) {
if (strcmp(argv[i], "-") == 0) {
in = STDIN_FILENO;
} else {
in = open(argv[i], O_RDONLY | O_BINARY);
if (in < 0) {
fprintf(stderr, "Cannot open input file %s\n", argv[i]);
exit(-1);
}
}
s = sparse_file_import(in, true, false);
if (!s) {
fprintf(stderr, "Failed to read sparse file\n");
exit(-1);
}
if (lseek(out, 0, SEEK_SET) == -1) {
perror("lseek failed");
exit(EXIT_FAILURE);
}
if (sparse_file_write(s, out, false, false, false) < 0) {
fprintf(stderr, "Cannot write output file\n");
exit(-1);
}
sparse_file_destroy(s);
close(in);
}
close(out);
exit(0);
}

86
simg2img.cpp Normal file
View File

@ -0,0 +1,86 @@
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <sparse/sparse.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#ifndef O_BINARY
#define O_BINARY 0
#endif
void usage() {
fprintf(stderr, "Usage: simg2img <sparse_image_files> <raw_image_file>\n");
}
int main(int argc, char* argv[]) {
int in;
int out;
int i;
struct sparse_file* s;
if (argc < 3) {
usage();
exit(-1);
}
out = open(argv[argc - 1], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
if (out < 0) {
fprintf(stderr, "Cannot open output file %s\n", argv[argc - 1]);
exit(-1);
}
for (i = 1; i < argc - 1; i++) {
if (strcmp(argv[i], "-") == 0) {
in = STDIN_FILENO;
} else {
in = open(argv[i], O_RDONLY | O_BINARY);
if (in < 0) {
fprintf(stderr, "Cannot open input file %s\n", argv[i]);
exit(-1);
}
}
s = sparse_file_import(in, true, false);
if (!s) {
fprintf(stderr, "Failed to read sparse file\n");
exit(-1);
}
if (lseek(out, 0, SEEK_SET) == -1) {
perror("lseek failed");
exit(EXIT_FAILURE);
}
if (sparse_file_write(s, out, false, false, false) < 0) {
fprintf(stderr, "Cannot write output file\n");
exit(-1);
}
sparse_file_destroy(s);
close(in);
}
close(out);
exit(0);
}

115
simg2simg.c
View File

@ -1,115 +0,0 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#define _GNU_SOURCE
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <sparse/sparse.h>
#ifndef O_BINARY
#define O_BINARY 0
#endif
void usage()
{
fprintf(stderr, "Usage: simg2simg <sparse image file> <sparse_image_file> <max_size>\n");
}
int main(int argc, char *argv[])
{
int in;
int out;
int i;
int ret;
struct sparse_file *s;
int64_t max_size;
struct sparse_file **out_s;
int files;
char filename[4096];
if (argc != 4) {
usage();
exit(-1);
}
max_size = atoll(argv[3]);
in = open(argv[1], O_RDONLY | O_BINARY);
if (in < 0) {
fprintf(stderr, "Cannot open input file %s\n", argv[1]);
exit(-1);
}
s = sparse_file_import(in, true, false);
if (!s) {
fprintf(stderr, "Failed to import sparse file\n");
exit(-1);
}
files = sparse_file_resparse(s, max_size, NULL, 0);
if (files < 0) {
fprintf(stderr, "Failed to resparse\n");
exit(-1);
}
out_s = calloc(sizeof(struct sparse_file *), files);
if (!out_s) {
fprintf(stderr, "Failed to allocate sparse file array\n");
exit(-1);
}
files = sparse_file_resparse(s, max_size, out_s, files);
if (files < 0) {
fprintf(stderr, "Failed to resparse\n");
exit(-1);
}
for (i = 0; i < files; i++) {
ret = snprintf(filename, sizeof(filename), "%s.%d", argv[2], i);
if (ret >= (int)sizeof(filename)) {
fprintf(stderr, "Filename too long\n");
exit(-1);
}
out = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
if (out < 0) {
fprintf(stderr, "Cannot open output file %s\n", argv[2]);
exit(-1);
}
ret = sparse_file_write(out_s[i], out, false, true, false);
if (ret) {
fprintf(stderr, "Failed to write sparse file\n");
exit(-1);
}
close(out);
}
close(in);
exit(0);
}

111
simg2simg.cpp Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <sparse/sparse.h>
#ifndef O_BINARY
#define O_BINARY 0
#endif
void usage() {
fprintf(stderr, "Usage: simg2simg <sparse image file> <sparse_image_file> <max_size>\n");
}
int main(int argc, char* argv[]) {
int in;
int out;
int i;
int ret;
struct sparse_file* s;
int64_t max_size;
struct sparse_file** out_s;
int files;
char filename[4096];
if (argc != 4) {
usage();
exit(-1);
}
max_size = atoll(argv[3]);
in = open(argv[1], O_RDONLY | O_BINARY);
if (in < 0) {
fprintf(stderr, "Cannot open input file %s\n", argv[1]);
exit(-1);
}
s = sparse_file_import(in, true, false);
if (!s) {
fprintf(stderr, "Failed to import sparse file\n");
exit(-1);
}
files = sparse_file_resparse(s, max_size, nullptr, 0);
if (files < 0) {
fprintf(stderr, "Failed to resparse\n");
exit(-1);
}
out_s = (struct sparse_file**) calloc(sizeof(struct sparse_file*), files);
if (!out_s) {
fprintf(stderr, "Failed to allocate sparse file array\n");
exit(-1);
}
files = sparse_file_resparse(s, max_size, out_s, files);
if (files < 0) {
fprintf(stderr, "Failed to resparse\n");
exit(-1);
}
for (i = 0; i < files; i++) {
ret = snprintf(filename, sizeof(filename), "%s.%d", argv[2], i);
if (ret >= (int)sizeof(filename)) {
fprintf(stderr, "Filename too long\n");
exit(-1);
}
out = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
if (out < 0) {
fprintf(stderr, "Cannot open output file %s\n", argv[2]);
exit(-1);
}
ret = sparse_file_write(out_s[i], out, false, true, false);
if (ret) {
fprintf(stderr, "Failed to write sparse file\n");
exit(-1);
}
close(out);
}
close(in);
exit(0);
}

28
simg_dump.py
View File

@ -15,7 +15,6 @@
# limitations under the License.
from __future__ import print_function
import binascii
import csv
import getopt
import hashlib
@ -48,7 +47,7 @@ def main():
opts, args = getopt.getopt(sys.argv[1:],
"vsc:",
["verbose", "showhash", "csvfile"])
except getopt.GetoptError as e:
except getopt.GetoptError, e:
print(e)
usage(me)
for o, a in opts:
@ -95,12 +94,12 @@ def main():
print("%s: %s: I only know about version 1.0, but this is version %u.%u"
% (me, path, major_version, minor_version))
continue
if file_hdr_sz < 28:
print("%s: %s: The file header size was expected to be at least 28, but is %u."
if file_hdr_sz != 28:
print("%s: %s: The file header size was expected to be 28, but is %u."
% (me, path, file_hdr_sz))
continue
if chunk_hdr_sz < 12:
print("%s: %s: The chunk header size was expected to be at least 12, but is %u."
if chunk_hdr_sz != 12:
print("%s: %s: The chunk header size was expected to be 12, but is %u."
% (me, path, chunk_hdr_sz))
continue
@ -110,11 +109,6 @@ def main():
if image_checksum != 0:
print("checksum=0x%08X" % (image_checksum))
if file_hdr_sz > 28:
header_extra = FH.read(file_hdr_sz - 28)
print("%s: Header extra bytes: %s"
% (me, binascii.hexlify(header_extra)))
if not output:
continue
@ -133,7 +127,7 @@ def main():
chunk_type = header[0]
chunk_sz = header[2]
total_sz = header[3]
data_sz = total_sz - chunk_hdr_sz
data_sz = total_sz - 12
curhash = ""
curtype = ""
curpos = FH.tell()
@ -141,10 +135,6 @@ def main():
if verbose > 0:
print("%4u %10u %10u %7u %7u" % (i, curpos, data_sz, offset, chunk_sz),
end=" ")
if chunk_hdr_sz > 12:
header_extra = FH.read(chunk_hdr_sz - 12)
print("[Extra bytes: %s]"
% (binascii.hexlify(header_extra)), end=" ")
if chunk_type == 0xCAC1:
if data_sz != (chunk_sz * blk_sz):
@ -227,11 +217,5 @@ def main():
sys.exit(0)
# Python 3 shim
try:
xrange
except NameError:
xrange = range
if __name__ == "__main__":
main()

97
simg_samsungify.d
View File

@ -1,97 +0,0 @@
/**
* Copyright (C) 2018 Vladimir Panteleev
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Description:
*
* Tweak a sparse-image file to conform to Samsung's proprietary
* format modifications.
*
* This allows creating sparse-image files directly flashable e.g. on
* SM-N910H with Odin/Heimdall.
*/
module simg_samsungify;
import std.exception;
import std.mmfile;
import std.stdio;
struct SparseHeader
{
uint magic = 0xed26ff3a;
ushort major_version = 1;
ushort minor_version = 0;
ushort file_hdr_sz;
ushort chunk_hdr_sz;
uint blk_sz;
uint total_blks;
uint total_chunks;
uint image_checksum;
}
struct ChunkHeader
{
ubyte[2] chunk_type;
ushort reserved1;
uint chunk_sz;
uint total_sz;
}
immutable uint[1] headerExtra = [0];
immutable uint[1] chunkExtra = [0x00007fcb];
void main(string[] args)
{
enforce(args.length == 3, "Usage: simg_samsungify IN-FILE-NAME OUT-FILE-NAME");
string inFileName = args[1];
string outFileName = args[2];
auto m = new MmFile(inFileName);
enforce(m.length >= SparseHeader.sizeof, "Not enough bytes for header");
auto p = cast(ubyte*)m[].ptr;
auto pHeader = cast(SparseHeader*)p;
enforce(pHeader.file_hdr_sz == SparseHeader.sizeof,
"Wrong file header size (already samsung-ified?)");
enforce(pHeader.chunk_hdr_sz == ChunkHeader.sizeof,
"Wrong chunk header size (already samsung-ified?)");
auto newHeader = *pHeader;
newHeader.file_hdr_sz += headerExtra.sizeof;
newHeader.chunk_hdr_sz += chunkExtra.sizeof;
auto o = File(outFileName, "wb");
o.rawWrite((&newHeader)[0..1]);
o.rawWrite(headerExtra[]);
auto end = p + m.length;
p += pHeader.file_hdr_sz;
while (p < end)
{
enforce(p + pHeader.chunk_hdr_sz <= end,
"Not enough bytes for chunk header");
auto pChunkHeader = cast(ChunkHeader*)p;
//writeln(*pChunkHeader);
enforce(p + pChunkHeader.total_sz <= end,
"Not enough bytes for chunk");
auto dataLength = pChunkHeader.total_sz - pHeader.chunk_hdr_sz;
auto newChunkHeader = *pChunkHeader;
newChunkHeader.total_sz += chunkExtra.sizeof;
o.rawWrite((&newChunkHeader)[0..1]);
o.rawWrite(chunkExtra);
p += pHeader.chunk_hdr_sz;
o.rawWrite(p[0..dataLength]);
p += dataLength;
}
}

378
sparse.c
View File

@ -1,378 +0,0 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <assert.h>
#include <stdlib.h>
#include <sparse/sparse.h>
#include "defs.h"
#include "sparse_file.h"
#include "output_file.h"
#include "backed_block.h"
#include "sparse_defs.h"
#include "sparse_format.h"
struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len)
{
struct sparse_file *s = calloc(sizeof(struct sparse_file), 1);
if (!s) {
return NULL;
}
s->backed_block_list = backed_block_list_new(block_size);
if (!s->backed_block_list) {
free(s);
return NULL;
}
s->block_size = block_size;
s->len = len;
return s;
}
void sparse_file_destroy(struct sparse_file *s)
{
backed_block_list_destroy(s->backed_block_list);
free(s);
}
int sparse_file_add_data(struct sparse_file *s, void *data, unsigned int len, unsigned int block)
{
return backed_block_add_data(s->backed_block_list, data, len, block);
}
int sparse_file_add_fill(struct sparse_file *s,
uint32_t fill_val, unsigned int len, unsigned int block)
{
return backed_block_add_fill(s->backed_block_list, fill_val, len, block);
}
int sparse_file_add_file(struct sparse_file *s,
const char *filename, int64_t file_offset, unsigned int len,
unsigned int block)
{
return backed_block_add_file(s->backed_block_list, filename, file_offset, len, block);
}
int sparse_file_add_fd(struct sparse_file *s,
int fd, int64_t file_offset, unsigned int len, unsigned int block)
{
return backed_block_add_fd(s->backed_block_list, fd, file_offset, len, block);
}
unsigned int sparse_count_chunks(struct sparse_file *s)
{
struct backed_block *bb;
unsigned int last_block = 0;
unsigned int chunks = 0;
for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) {
if (backed_block_block(bb) > last_block) {
/* If there is a gap between chunks, add a skip chunk */
chunks++;
}
chunks++;
last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), s->block_size);
}
if (last_block < DIV_ROUND_UP(s->len, s->block_size)) {
chunks++;
}
return chunks;
}
static int sparse_file_write_block(struct output_file *out, struct backed_block *bb)
{
int ret = -EINVAL;
switch (backed_block_type(bb)) {
case BACKED_BLOCK_DATA:
ret = write_data_chunk(out, backed_block_len(bb), backed_block_data(bb));
break;
case BACKED_BLOCK_FILE:
ret = write_file_chunk(out, backed_block_len(bb),
backed_block_filename(bb), backed_block_file_offset(bb));
break;
case BACKED_BLOCK_FD:
ret = write_fd_chunk(out, backed_block_len(bb),
backed_block_fd(bb), backed_block_file_offset(bb));
break;
case BACKED_BLOCK_FILL:
ret = write_fill_chunk(out, backed_block_len(bb), backed_block_fill_val(bb));
break;
}
return ret;
}
static int write_all_blocks(struct sparse_file *s, struct output_file *out)
{
struct backed_block *bb;
unsigned int last_block = 0;
int64_t pad;
int ret = 0;
for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) {
if (backed_block_block(bb) > last_block) {
unsigned int blocks = backed_block_block(bb) - last_block;
write_skip_chunk(out, (int64_t) blocks * s->block_size);
}
ret = sparse_file_write_block(out, bb);
if (ret)
return ret;
last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), s->block_size);
}
pad = s->len - (int64_t) last_block *s->block_size;
assert(pad >= 0);
if (pad > 0) {
write_skip_chunk(out, pad);
}
return 0;
}
int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse, bool crc)
{
int ret;
int chunks;
struct output_file *out;
chunks = sparse_count_chunks(s);
out = output_file_open_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc);
if (!out)
return -ENOMEM;
ret = write_all_blocks(s, out);
output_file_close(out);
return ret;
}
int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc,
int (*write) (void *priv, const void *data, int len), void *priv)
{
int ret;
int chunks;
struct output_file *out;
chunks = sparse_count_chunks(s);
out = output_file_open_callback(write, priv, s->block_size, s->len, false, sparse, chunks, crc);
if (!out)
return -ENOMEM;
ret = write_all_blocks(s, out);
output_file_close(out);
return ret;
}
struct chunk_data {
void *priv;
unsigned int block;
unsigned int nr_blocks;
int (*write)(void *priv, const void *data, int len, unsigned int block,
unsigned int nr_blocks);
};
static int foreach_chunk_write(void *priv, const void *data, int len)
{
struct chunk_data *chk = priv;
return chk->write(chk->priv, data, len, chk->block, chk->nr_blocks);
}
int sparse_file_foreach_chunk(struct sparse_file *s, bool sparse, bool crc,
int (*write)(void *priv, const void *data, int len, unsigned int block,
unsigned int nr_blocks),
void *priv)
{
int ret;
int chunks;
struct chunk_data chk;
struct output_file *out;
struct backed_block *bb;
chk.priv = priv;
chk.write = write;
chk.block = chk.nr_blocks = 0;
chunks = sparse_count_chunks(s);
out = output_file_open_callback(foreach_chunk_write, &chk,
s->block_size, s->len, false, sparse,
chunks, crc);
if (!out)
return -ENOMEM;
for (bb = backed_block_iter_new(s->backed_block_list); bb;
bb = backed_block_iter_next(bb)) {
chk.block = backed_block_block(bb);
chk.nr_blocks = (backed_block_len(bb) - 1) / s->block_size + 1;
ret = sparse_file_write_block(out, bb);
if (ret)
return ret;
}
output_file_close(out);
return ret;
}
static int out_counter_write(void *priv, const void *data __unused, int len)
{
int64_t *count = priv;
*count += len;
return 0;
}
int64_t sparse_file_len(struct sparse_file * s, bool sparse, bool crc)
{
int ret;
int chunks = sparse_count_chunks(s);
int64_t count = 0;
struct output_file *out;
out = output_file_open_callback(out_counter_write, &count,
s->block_size, s->len, false, sparse, chunks, crc);
if (!out) {
return -1;
}
ret = write_all_blocks(s, out);
output_file_close(out);
if (ret < 0) {
return -1;
}
return count;
}
unsigned int sparse_file_block_size(struct sparse_file *s)
{
return s->block_size;
}
static struct backed_block *move_chunks_up_to_len(struct sparse_file *from,
struct sparse_file *to, unsigned int len)
{
int64_t count = 0;
struct output_file *out_counter;
struct backed_block *last_bb = NULL;
struct backed_block *bb;
struct backed_block *start;
unsigned int last_block = 0;
int64_t file_len = 0;
int ret;
/*
* overhead is sparse file header, the potential end skip
* chunk and crc chunk.
*/
int overhead = sizeof(sparse_header_t) + 2 * sizeof(chunk_header_t) + sizeof(uint32_t);
len -= overhead;
start = backed_block_iter_new(from->backed_block_list);
out_counter = output_file_open_callback(out_counter_write, &count,
to->block_size, to->len, false, true, 0, false);
if (!out_counter) {
return NULL;
}
for (bb = start; bb; bb = backed_block_iter_next(bb)) {
count = 0;
if (backed_block_block(bb) > last_block)
count += sizeof(chunk_header_t);
last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), to->block_size);
/* will call out_counter_write to update count */
ret = sparse_file_write_block(out_counter, bb);
if (ret) {
bb = NULL;
goto out;
}
if (file_len + count > len) {
/*
* If the remaining available size is more than 1/8th of the
* requested size, split the chunk. Results in sparse files that
* are at least 7/8ths of the requested size
*/
file_len += sizeof(chunk_header_t);
if (!last_bb || (len - file_len > (len / 8))) {
backed_block_split(from->backed_block_list, bb, len - file_len);
last_bb = bb;
}
goto move;
}
file_len += count;
last_bb = bb;
}
move:
backed_block_list_move(from->backed_block_list, to->backed_block_list, start, last_bb);
out:
output_file_close(out_counter);
return bb;
}
int sparse_file_resparse(struct sparse_file *in_s, unsigned int max_len,
struct sparse_file **out_s, int out_s_count)
{
struct backed_block *bb;
struct sparse_file *s;
struct sparse_file *tmp;
int c = 0;
tmp = sparse_file_new(in_s->block_size, in_s->len);
if (!tmp) {
return -ENOMEM;
}
do {
s = sparse_file_new(in_s->block_size, in_s->len);
bb = move_chunks_up_to_len(in_s, s, max_len);
if (c < out_s_count) {
out_s[c] = s;
} else {
backed_block_list_move(s->backed_block_list, tmp->backed_block_list, NULL, NULL);
sparse_file_destroy(s);
}
c++;
} while (bb);
backed_block_list_move(tmp->backed_block_list, in_s->backed_block_list, NULL, NULL);
sparse_file_destroy(tmp);
return c;
}
void sparse_file_verbose(struct sparse_file *s)
{
s->verbose = true;
}

348
sparse.cpp Normal file
View File

@ -0,0 +1,348 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <assert.h>
#include <stdlib.h>
#include <sparse/sparse.h>
#include "defs.h"
#include "sparse_file.h"
#include "backed_block.h"
#include "output_file.h"
#include "sparse_defs.h"
#include "sparse_format.h"
struct sparse_file* sparse_file_new(unsigned int block_size, int64_t len) {
struct sparse_file* s = reinterpret_cast<sparse_file*>(calloc(sizeof(struct sparse_file), 1));
if (!s) {
return nullptr;
}
s->backed_block_list = backed_block_list_new(block_size);
if (!s->backed_block_list) {
free(s);
return nullptr;
}
s->block_size = block_size;
s->len = len;
return s;
}
void sparse_file_destroy(struct sparse_file* s) {
backed_block_list_destroy(s->backed_block_list);
free(s);
}
int sparse_file_add_data(struct sparse_file* s, void* data, unsigned int len, unsigned int block) {
return backed_block_add_data(s->backed_block_list, data, len, block);
}
int sparse_file_add_fill(struct sparse_file* s, uint32_t fill_val, unsigned int len,
unsigned int block) {
return backed_block_add_fill(s->backed_block_list, fill_val, len, block);
}
int sparse_file_add_file(struct sparse_file* s, const char* filename, int64_t file_offset,
unsigned int len, unsigned int block) {
return backed_block_add_file(s->backed_block_list, filename, file_offset, len, block);
}
int sparse_file_add_fd(struct sparse_file* s, int fd, int64_t file_offset, unsigned int len,
unsigned int block) {
return backed_block_add_fd(s->backed_block_list, fd, file_offset, len, block);
}
unsigned int sparse_count_chunks(struct sparse_file* s) {
struct backed_block* bb;
unsigned int last_block = 0;
unsigned int chunks = 0;
for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) {
if (backed_block_block(bb) > last_block) {
/* If there is a gap between chunks, add a skip chunk */
chunks++;
}
chunks++;
last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), s->block_size);
}
if (last_block < DIV_ROUND_UP(s->len, s->block_size)) {
chunks++;
}
return chunks;
}
static int sparse_file_write_block(struct output_file* out, struct backed_block* bb) {
int ret = -EINVAL;
switch (backed_block_type(bb)) {
case BACKED_BLOCK_DATA:
ret = write_data_chunk(out, backed_block_len(bb), backed_block_data(bb));
break;
case BACKED_BLOCK_FILE:
ret = write_file_chunk(out, backed_block_len(bb), backed_block_filename(bb),
backed_block_file_offset(bb));
break;
case BACKED_BLOCK_FD:
ret = write_fd_chunk(out, backed_block_len(bb), backed_block_fd(bb),
backed_block_file_offset(bb));
break;
case BACKED_BLOCK_FILL:
ret = write_fill_chunk(out, backed_block_len(bb), backed_block_fill_val(bb));
break;
}
return ret;
}
static int write_all_blocks(struct sparse_file* s, struct output_file* out) {
struct backed_block* bb;
unsigned int last_block = 0;
int64_t pad;
int ret = 0;
for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) {
if (backed_block_block(bb) > last_block) {
unsigned int blocks = backed_block_block(bb) - last_block;
write_skip_chunk(out, (int64_t)blocks * s->block_size);
}
ret = sparse_file_write_block(out, bb);
if (ret) return ret;
last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), s->block_size);
}
pad = s->len - (int64_t)last_block * s->block_size;
assert(pad >= 0);
if (pad > 0) {
write_skip_chunk(out, pad);
}
return 0;
}
int sparse_file_write(struct sparse_file* s, int fd, bool gz, bool sparse, bool crc) {
int ret;
int chunks;
struct output_file* out;
chunks = sparse_count_chunks(s);
out = output_file_open_fd(fd, s->block_size, s->len, gz, sparse, chunks, crc);
if (!out) return -ENOMEM;
ret = write_all_blocks(s, out);
output_file_close(out);
return ret;
}
int sparse_file_callback(struct sparse_file* s, bool sparse, bool crc,
int (*write)(void* priv, const void* data, size_t len), void* priv) {
int ret;
int chunks;
struct output_file* out;
chunks = sparse_count_chunks(s);
out = output_file_open_callback(write, priv, s->block_size, s->len, false, sparse, chunks, crc);
if (!out) return -ENOMEM;
ret = write_all_blocks(s, out);
output_file_close(out);
return ret;
}
struct chunk_data {
void* priv;
unsigned int block;
unsigned int nr_blocks;
int (*write)(void* priv, const void* data, size_t len, unsigned int block, unsigned int nr_blocks);
};
static int foreach_chunk_write(void* priv, const void* data, size_t len) {
struct chunk_data* chk = reinterpret_cast<chunk_data*>(priv);
return chk->write(chk->priv, data, len, chk->block, chk->nr_blocks);
}
int sparse_file_foreach_chunk(struct sparse_file* s, bool sparse, bool crc,
int (*write)(void* priv, const void* data, size_t len,
unsigned int block, unsigned int nr_blocks),
void* priv) {
int ret;
int chunks;
struct chunk_data chk;
struct output_file* out;
struct backed_block* bb;
chk.priv = priv;
chk.write = write;
chk.block = chk.nr_blocks = 0;
chunks = sparse_count_chunks(s);
out = output_file_open_callback(foreach_chunk_write, &chk, s->block_size, s->len, false, sparse,
chunks, crc);
if (!out) return -ENOMEM;
for (bb = backed_block_iter_new(s->backed_block_list); bb; bb = backed_block_iter_next(bb)) {
chk.block = backed_block_block(bb);
chk.nr_blocks = (backed_block_len(bb) - 1) / s->block_size + 1;
ret = sparse_file_write_block(out, bb);
if (ret) return ret;
}
output_file_close(out);
return ret;
}
static int out_counter_write(void* priv, const void* data __unused, size_t len) {
int64_t* count = reinterpret_cast<int64_t*>(priv);
*count += len;
return 0;
}
int64_t sparse_file_len(struct sparse_file* s, bool sparse, bool crc) {
int ret;
int chunks = sparse_count_chunks(s);
int64_t count = 0;
struct output_file* out;
out = output_file_open_callback(out_counter_write, &count, s->block_size, s->len, false, sparse,
chunks, crc);
if (!out) {
return -1;
}
ret = write_all_blocks(s, out);
output_file_close(out);
if (ret < 0) {
return -1;
}
return count;
}
unsigned int sparse_file_block_size(struct sparse_file* s) {
return s->block_size;
}
static struct backed_block* move_chunks_up_to_len(struct sparse_file* from, struct sparse_file* to,
unsigned int len) {
int64_t count = 0;
struct output_file* out_counter;
struct backed_block* last_bb = nullptr;
struct backed_block* bb;
struct backed_block* start;
unsigned int last_block = 0;
int64_t file_len = 0;
int ret;
/*
* overhead is sparse file header, the potential end skip
* chunk and crc chunk.
*/
int overhead = sizeof(sparse_header_t) + 2 * sizeof(chunk_header_t) + sizeof(uint32_t);
len -= overhead;
start = backed_block_iter_new(from->backed_block_list);
out_counter = output_file_open_callback(out_counter_write, &count, to->block_size, to->len, false,
true, 0, false);
if (!out_counter) {
return nullptr;
}
for (bb = start; bb; bb = backed_block_iter_next(bb)) {
count = 0;
if (backed_block_block(bb) > last_block) count += sizeof(chunk_header_t);
last_block = backed_block_block(bb) + DIV_ROUND_UP(backed_block_len(bb), to->block_size);
/* will call out_counter_write to update count */
ret = sparse_file_write_block(out_counter, bb);
if (ret) {
bb = nullptr;
goto out;
}
if (file_len + count > len) {
/*
* If the remaining available size is more than 1/8th of the
* requested size, split the chunk. Results in sparse files that
* are at least 7/8ths of the requested size
*/
file_len += sizeof(chunk_header_t);
if (!last_bb || (len - file_len > (len / 8))) {
backed_block_split(from->backed_block_list, bb, len - file_len);
last_bb = bb;
}
goto move;
}
file_len += count;
last_bb = bb;
}
move:
backed_block_list_move(from->backed_block_list, to->backed_block_list, start, last_bb);
out:
output_file_close(out_counter);
return bb;
}
int sparse_file_resparse(struct sparse_file* in_s, unsigned int max_len, struct sparse_file** out_s,
int out_s_count) {
struct backed_block* bb;
struct sparse_file* s;
struct sparse_file* tmp;
int c = 0;
tmp = sparse_file_new(in_s->block_size, in_s->len);
if (!tmp) {
return -ENOMEM;
}
do {
s = sparse_file_new(in_s->block_size, in_s->len);
bb = move_chunks_up_to_len(in_s, s, max_len);
if (c < out_s_count) {
out_s[c] = s;
} else {
backed_block_list_move(s->backed_block_list, tmp->backed_block_list, nullptr, nullptr);
sparse_file_destroy(s);
}
c++;
} while (bb);
backed_block_list_move(tmp->backed_block_list, in_s->backed_block_list, nullptr, nullptr);
sparse_file_destroy(tmp);
return c;
}
void sparse_file_verbose(struct sparse_file* s) {
s->verbose = true;
}

91
sparse_crc32.c → sparse_crc32.cpp
View File

@ -44,52 +44,41 @@
/* Code taken from FreeBSD 8 */
#include <stdint.h>
#include <stdio.h>
static uint32_t crc32_tab[] = {
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106,
0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
};
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d};
/*
* A function that calculates the CRC-32 based on the table above is
@ -98,13 +87,11 @@ static uint32_t crc32_tab[] = {
* in sys/libkern.h, where it can be inlined.
*/
uint32_t sparse_crc32(uint32_t crc_in, const void *buf, int size)
{
const uint8_t *p = buf;
uint32_t crc;
uint32_t sparse_crc32(uint32_t crc_in, const void* buf, size_t size) {
const uint8_t* p = reinterpret_cast<const uint8_t*>(buf);
uint32_t crc;
crc = crc_in ^ ~0U;
while (size--)
crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8);
return crc ^ ~0U;
crc = crc_in ^ ~0U;
while (size--) crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8);
return crc ^ ~0U;
}

9
sparse_crc32.h
View File

@ -19,13 +19,6 @@
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
uint32_t sparse_crc32(uint32_t crc, const void* buf, size_t size);
uint32_t sparse_crc32(uint32_t crc, const void *buf, size_t size);
#ifdef __cplusplus
}
#endif
#endif

9
sparse_defs.h
View File

@ -39,11 +39,14 @@ typedef unsigned int u32;
typedef unsigned short int u16;
typedef unsigned char u8;
#define DIV_ROUND_UP(x, y) (((x) + (y) - 1)/(y))
#define ALIGN(x, y) ((y) * DIV_ROUND_UP((x), (y)))
#define DIV_ROUND_UP(x, y) (((x) + (y)-1) / (y))
#define ALIGN(x, y) ((y)*DIV_ROUND_UP((x), (y)))
#define ALIGN_DOWN(x, y) ((y) * ((x) / (y)))
#define error(fmt, args...) do { fprintf(stderr, "error: %s: " fmt "\n", __func__, ## args); } while (0)
#define error(fmt, args...) \
do { \
fprintf(stderr, "error: %s: " fmt "\n", __func__, ##args); \
} while (0)
#define error_errno(s, args...) error(s ": %s", ##args, strerror(errno))
#endif

15
sparse_err.c → sparse_err.cpp
View File

@ -20,14 +20,13 @@
#include <stdio.h>
#include <unistd.h>
void sparse_default_print(const char *fmt, ...)
{
va_list argp;
void sparse_default_print(const char* fmt, ...) {
va_list argp;
va_start(argp, fmt);
vfprintf(stderr, fmt, argp);
va_end(argp);
va_start(argp, fmt);
vfprintf(stderr, fmt, argp);
va_end(argp);
}
void (*sparse_print_error) (const char *fmt, ...) = sparse_default_print;
void (*sparse_print_verbose) (const char *fmt, ...) = sparse_default_print;
void (*sparse_print_error)(const char* fmt, ...) = sparse_default_print;
void (*sparse_print_verbose)(const char* fmt, ...) = sparse_default_print;

20
sparse_file.h
View File

@ -17,15 +17,23 @@
#ifndef _LIBSPARSE_SPARSE_FILE_H_
#define _LIBSPARSE_SPARSE_FILE_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <sparse/sparse.h>
struct sparse_file {
unsigned int block_size;
int64_t len;
bool verbose;
unsigned int block_size;
int64_t len;
bool verbose;
struct backed_block_list *backed_block_list;
struct output_file *out;
struct backed_block_list* backed_block_list;
struct output_file* out;
};
#endif /* _LIBSPARSE_SPARSE_FILE_H_ */
#ifdef __cplusplus
}
#endif
#endif /* _LIBSPARSE_SPARSE_FILE_H_ */

48
sparse_format.h
View File

@ -18,32 +18,36 @@
#define _LIBSPARSE_SPARSE_FORMAT_H_
#include "sparse_defs.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct sparse_header {
__le32 magic; /* 0xed26ff3a */
__le16 major_version; /* (0x1) - reject images with higher major versions */
__le16 minor_version; /* (0x0) - allow images with higer minor versions */
__le16 file_hdr_sz; /* 28 bytes for first revision of the file format */
__le16 chunk_hdr_sz; /* 12 bytes for first revision of the file format */
__le32 blk_sz; /* block size in bytes, must be a multiple of 4 (4096) */
__le32 total_blks; /* total blocks in the non-sparse output image */
__le32 total_chunks; /* total chunks in the sparse input image */
__le32 image_checksum; /* CRC32 checksum of the original data, counting "don't care" */
/* as 0. Standard 802.3 polynomial, use a Public Domain */
/* table implementation */
__le32 magic; /* 0xed26ff3a */
__le16 major_version; /* (0x1) - reject images with higher major versions */
__le16 minor_version; /* (0x0) - allow images with higer minor versions */
__le16 file_hdr_sz; /* 28 bytes for first revision of the file format */
__le16 chunk_hdr_sz; /* 12 bytes for first revision of the file format */
__le32 blk_sz; /* block size in bytes, must be a multiple of 4 (4096) */
__le32 total_blks; /* total blocks in the non-sparse output image */
__le32 total_chunks; /* total chunks in the sparse input image */
__le32 image_checksum; /* CRC32 checksum of the original data, counting "don't care" */
/* as 0. Standard 802.3 polynomial, use a Public Domain */
/* table implementation */
} sparse_header_t;
#define SPARSE_HEADER_MAGIC 0xed26ff3a
#define SPARSE_HEADER_MAGIC 0xed26ff3a
#define CHUNK_TYPE_RAW 0xCAC1
#define CHUNK_TYPE_FILL 0xCAC2
#define CHUNK_TYPE_DONT_CARE 0xCAC3
#define CHUNK_TYPE_CRC32 0xCAC4
#define CHUNK_TYPE_RAW 0xCAC1
#define CHUNK_TYPE_FILL 0xCAC2
#define CHUNK_TYPE_DONT_CARE 0xCAC3
#define CHUNK_TYPE_CRC32 0xCAC4
typedef struct chunk_header {
__le16 chunk_type; /* 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care */
__le16 reserved1;
__le32 chunk_sz; /* in blocks in output image */
__le32 total_sz; /* in bytes of chunk input file including chunk header and data */
__le16 chunk_type; /* 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care */
__le16 reserved1;
__le32 chunk_sz; /* in blocks in output image */
__le32 total_sz; /* in bytes of chunk input file including chunk header and data */
} chunk_header_t;
/* Following a Raw or Fill or CRC32 chunk is data.
@ -52,4 +56,8 @@ typedef struct chunk_header {
* For a CRC32 chunk, it's 4 bytes of CRC32
*/
#ifdef __cplusplus
}
#endif
#endif

506
sparse_read.c
View File

@ -1,506 +0,0 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _GNU_SOURCE
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <inttypes.h>
#include <fcntl.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sparse/sparse.h>
#include "defs.h"
#include "output_file.h"
#include "sparse_crc32.h"
#include "sparse_file.h"
#include "sparse_format.h"
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t
#endif
#define SPARSE_HEADER_MAJOR_VER 1
#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
#define COPY_BUF_SIZE (1024U*1024U)
static char *copybuf;
#define min(a, b) \
({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
static void verbose_error(bool verbose, int err, const char *fmt, ...)
{
char *s = "";
char *at = "";
if (fmt) {
va_list argp;
int size;
va_start(argp, fmt);
size = vsnprintf(NULL, 0, fmt, argp);
va_end(argp);
if (size < 0) {
return;
}
at = malloc(size + 1);
if (at == NULL) {
return;
}
va_start(argp, fmt);
vsnprintf(at, size, fmt, argp);
va_end(argp);
at[size] = 0;
s = " at ";
}
if (verbose) {
#ifndef USE_MINGW
if (err == -EOVERFLOW) {
sparse_print_verbose("EOF while reading file%s%s\n", s, at);
} else
#endif
if (err == -EINVAL) {
sparse_print_verbose("Invalid sparse file format%s%s\n", s, at);
} else if (err == -ENOMEM) {
sparse_print_verbose("Failed allocation while reading file%s%s\n", s, at);
} else {
sparse_print_verbose("Unknown error %d%s%s\n", err, s, at);
}
}
if (fmt) {
free(at);
}
}
static int process_raw_chunk(struct sparse_file *s, unsigned int chunk_size,
int fd, int64_t offset, unsigned int blocks, unsigned int block,
uint32_t * crc32)
{
int ret;
int chunk;
unsigned int len = blocks * s->block_size;
if (chunk_size % s->block_size != 0) {
return -EINVAL;
}
if (chunk_size / s->block_size != blocks) {
return -EINVAL;
}
ret = sparse_file_add_fd(s, fd, offset, len, block);
if (ret < 0) {
return ret;
}
if (crc32) {
while (len) {
chunk = min(len, COPY_BUF_SIZE);
ret = read_all(fd, copybuf, chunk);
if (ret < 0) {
return ret;
}
*crc32 = sparse_crc32(*crc32, copybuf, chunk);
len -= chunk;
}
} else {
lseek64(fd, len, SEEK_CUR);
}
return 0;
}
static int process_fill_chunk(struct sparse_file *s, unsigned int chunk_size,
int fd, unsigned int blocks, unsigned int block, uint32_t * crc32)
{
int ret;
int chunk;
int64_t len = (int64_t) blocks * s->block_size;
uint32_t fill_val;
uint32_t *fillbuf;
unsigned int i;
if (chunk_size != sizeof(fill_val)) {
return -EINVAL;
}
ret = read_all(fd, &fill_val, sizeof(fill_val));
if (ret < 0) {
return ret;
}
ret = sparse_file_add_fill(s, fill_val, len, block);
if (ret < 0) {
return ret;
}
if (crc32) {
/* Fill copy_buf with the fill value */
fillbuf = (uint32_t *) copybuf;
for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
fillbuf[i] = fill_val;
}
while (len) {
chunk = min(len, COPY_BUF_SIZE);
*crc32 = sparse_crc32(*crc32, copybuf, chunk);
len -= chunk;
}
}
return 0;
}
static int process_skip_chunk(struct sparse_file *s, unsigned int chunk_size,
int fd __unused, unsigned int blocks,
unsigned int block __unused, uint32_t * crc32)
{
if (chunk_size != 0) {
return -EINVAL;
}
if (crc32) {
int64_t len = (int64_t) blocks * s->block_size;
memset(copybuf, 0, COPY_BUF_SIZE);
while (len) {
int chunk = min(len, COPY_BUF_SIZE);
*crc32 = sparse_crc32(*crc32, copybuf, chunk);
len -= chunk;
}
}
return 0;
}
static int process_crc32_chunk(int fd, unsigned int chunk_size, uint32_t * crc32)
{
uint32_t file_crc32;
int ret;
if (chunk_size != sizeof(file_crc32)) {
return -EINVAL;
}
ret = read_all(fd, &file_crc32, sizeof(file_crc32));
if (ret < 0) {
return ret;
}
if (crc32 != NULL && file_crc32 != *crc32) {
return -EINVAL;
}
return 0;
}
static int process_chunk(struct sparse_file *s, int fd, off64_t offset,
unsigned int chunk_hdr_sz, chunk_header_t * chunk_header,
unsigned int cur_block, uint32_t * crc_ptr)
{
int ret;
unsigned int chunk_data_size;
chunk_data_size = chunk_header->total_sz - chunk_hdr_sz;
switch (chunk_header->chunk_type) {
case CHUNK_TYPE_RAW:
ret = process_raw_chunk(s, chunk_data_size, fd, offset,
chunk_header->chunk_sz, cur_block, crc_ptr);
if (ret < 0) {
verbose_error(s->verbose, ret, "data block at %" PRId64, offset);
return ret;
}
return chunk_header->chunk_sz;
case CHUNK_TYPE_FILL:
ret = process_fill_chunk(s, chunk_data_size, fd,
chunk_header->chunk_sz, cur_block, crc_ptr);
if (ret < 0) {
verbose_error(s->verbose, ret, "fill block at %" PRId64, offset);
return ret;
}
return chunk_header->chunk_sz;
case CHUNK_TYPE_DONT_CARE:
ret = process_skip_chunk(s, chunk_data_size, fd,
chunk_header->chunk_sz, cur_block, crc_ptr);
if (chunk_data_size != 0) {
if (ret < 0) {
verbose_error(s->verbose, ret, "skip block at %" PRId64, offset);
return ret;
}
}
return chunk_header->chunk_sz;
case CHUNK_TYPE_CRC32:
ret = process_crc32_chunk(fd, chunk_data_size, crc_ptr);
if (ret < 0) {
verbose_error(s->verbose, -EINVAL, "crc block at %" PRId64, offset);
return ret;
}
return 0;
default:
verbose_error(s->verbose, -EINVAL, "unknown block %04X at %" PRId64,
chunk_header->chunk_type, offset);
}
return 0;
}
static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc)
{
int ret;
unsigned int i;
sparse_header_t sparse_header;
chunk_header_t chunk_header;
uint32_t crc32 = 0;
uint32_t *crc_ptr = 0;
unsigned int cur_block = 0;
off64_t offset;
if (!copybuf) {
copybuf = malloc(COPY_BUF_SIZE);
}
if (!copybuf) {
return -ENOMEM;
}
if (crc) {
crc_ptr = &crc32;
}
ret = read_all(fd, &sparse_header, sizeof(sparse_header));
if (ret < 0) {
return ret;
}
if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
return -EINVAL;
}
if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
return -EINVAL;
}
if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
return -EINVAL;
}
if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) {
return -EINVAL;
}
if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
/* Skip the remaining bytes in a header that is longer than
* we expected.
*/
lseek64(fd, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR);
}
for (i = 0; i < sparse_header.total_chunks; i++) {
ret = read_all(fd, &chunk_header, sizeof(chunk_header));
if (ret < 0) {
return ret;
}
if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
/* Skip the remaining bytes in a header that is longer than
* we expected.
*/
lseek64(fd, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR);
}
offset = lseek64(fd, 0, SEEK_CUR);
ret = process_chunk(s, fd, offset, sparse_header.chunk_hdr_sz, &chunk_header,
cur_block, crc_ptr);
if (ret < 0) {
return ret;
}
cur_block += ret;
}
if (sparse_header.total_blks != cur_block) {
return -EINVAL;
}
return 0;
}
static int sparse_file_read_normal(struct sparse_file *s, int fd)
{
int ret;
uint32_t *buf = malloc(s->block_size);
unsigned int block = 0;
int64_t remain = s->len;
int64_t offset = 0;
unsigned int to_read;
unsigned int i;
bool sparse_block;
if (!buf) {
return -ENOMEM;
}
while (remain > 0) {
to_read = min(remain, s->block_size);
ret = read_all(fd, buf, to_read);
if (ret < 0) {
error("failed to read sparse file");
free(buf);
return ret;
}
if (to_read == s->block_size) {
sparse_block = true;
for (i = 1; i < s->block_size / sizeof(uint32_t); i++) {
if (buf[0] != buf[i]) {
sparse_block = false;
break;
}
}
} else {
sparse_block = false;
}
if (sparse_block) {
/* TODO: add flag to use skip instead of fill for buf[0] == 0 */
sparse_file_add_fill(s, buf[0], to_read, block);
} else {
sparse_file_add_fd(s, fd, offset, to_read, block);
}
remain -= to_read;
offset += to_read;
block++;
}
free(buf);
return 0;
}
int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
{
if (crc && !sparse) {
return -EINVAL;
}
if (sparse) {
return sparse_file_read_sparse(s, fd, crc);
} else {
return sparse_file_read_normal(s, fd);
}
}
struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc)
{
int ret;
sparse_header_t sparse_header;
int64_t len;
struct sparse_file *s;
ret = read_all(fd, &sparse_header, sizeof(sparse_header));
if (ret < 0) {
verbose_error(verbose, ret, "header");
return NULL;
}
if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
verbose_error(verbose, -EINVAL, "header magic");
return NULL;
}
if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
verbose_error(verbose, -EINVAL, "header major version");
return NULL;
}
if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
return NULL;
}
if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) {
return NULL;
}
len = (int64_t) sparse_header.total_blks * sparse_header.blk_sz;
s = sparse_file_new(sparse_header.blk_sz, len);
if (!s) {
verbose_error(verbose, -EINVAL, NULL);
return NULL;
}
ret = lseek64(fd, 0, SEEK_SET);
if (ret < 0) {
verbose_error(verbose, ret, "seeking");
sparse_file_destroy(s);
return NULL;
}
s->verbose = verbose;
ret = sparse_file_read(s, fd, true, crc);
if (ret < 0) {
sparse_file_destroy(s);
return NULL;
}
return s;
}
struct sparse_file *sparse_file_import_auto(int fd, bool crc, bool verbose)
{
struct sparse_file *s;
int64_t len;
int ret;
s = sparse_file_import(fd, verbose, crc);
if (s) {
return s;
}
len = lseek64(fd, 0, SEEK_END);
if (len < 0) {
return NULL;
}
lseek64(fd, 0, SEEK_SET);
s = sparse_file_new(4096, len);
if (!s) {
return NULL;
}
ret = sparse_file_read_normal(s, fd);
if (ret < 0) {
sparse_file_destroy(s);
return NULL;
}
return s;
}

577
sparse_read.cpp Normal file
View File

@ -0,0 +1,577 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE 1
#include <fcntl.h>
#include <inttypes.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <algorithm>
#include <string>
#include <sparse/sparse.h>
#include "android-base/stringprintf.h"
#include "defs.h"
#include "output_file.h"
#include "sparse_crc32.h"
#include "sparse_file.h"
#include "sparse_format.h"
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t
#endif
#define SPARSE_HEADER_MAJOR_VER 1
#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
static constexpr int64_t COPY_BUF_SIZE = 1024 * 1024;
static char* copybuf;
static std::string ErrorString(int err) {
if (err == -EOVERFLOW) return "EOF while reading file";
if (err == -EINVAL) return "Invalid sparse file format";
if (err == -ENOMEM) return "Failed allocation while reading file";
return android::base::StringPrintf("Unknown error %d", err);
}
class SparseFileSource {
public:
/* Seeks the source ahead by the given offset. */
virtual void Seek(int64_t offset) = 0;
/* Return the current offset. */
virtual int64_t GetOffset() = 0;
/* Set the current offset. Return 0 if successful. */
virtual int SetOffset(int64_t offset) = 0;
/* Adds the given length from the current offset of the source to the file at the given block.
* Return 0 if successful. */
virtual int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) = 0;
/* Get data of fixed size from the current offset and seek len bytes. Return 0 if successful. */
virtual int ReadValue(void* ptr, int len) = 0;
/* Find the crc32 of the next len bytes and seek ahead len bytes. Return 0 if successful. */
virtual int GetCrc32(uint32_t* crc32, int64_t len) = 0;
virtual ~SparseFileSource(){};
};
class SparseFileFdSource : public SparseFileSource {
private:
int fd;
public:
SparseFileFdSource(int fd) : fd(fd) {}
~SparseFileFdSource() override {}
void Seek(int64_t off) override { lseek64(fd, off, SEEK_CUR); }
int64_t GetOffset() override { return lseek64(fd, 0, SEEK_CUR); }
int SetOffset(int64_t offset) override {
return lseek64(fd, offset, SEEK_SET) == offset ? 0 : -errno;
}
int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) override {
return sparse_file_add_fd(s, fd, GetOffset(), len, block);
}
int ReadValue(void* ptr, int len) override { return read_all(fd, ptr, len); }
int GetCrc32(uint32_t* crc32, int64_t len) override {
int chunk;
int ret;
while (len) {
chunk = std::min(len, COPY_BUF_SIZE);
ret = read_all(fd, copybuf, chunk);
if (ret < 0) {
return ret;
}
*crc32 = sparse_crc32(*crc32, copybuf, chunk);
len -= chunk;
}
return 0;
}
};
class SparseFileBufSource : public SparseFileSource {
private:
char* buf;
int64_t offset;
public:
SparseFileBufSource(char* buf) : buf(buf), offset(0) {}
~SparseFileBufSource() override {}
void Seek(int64_t off) override {
buf += off;
offset += off;
}
int64_t GetOffset() override { return offset; }
int SetOffset(int64_t off) override {
buf += off - offset;
offset = off;
return 0;
}
int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) override {
return sparse_file_add_data(s, buf, len, block);
}
int ReadValue(void* ptr, int len) override {
memcpy(ptr, buf, len);
Seek(len);
return 0;
}
int GetCrc32(uint32_t* crc32, int64_t len) override {
*crc32 = sparse_crc32(*crc32, buf, len);
Seek(len);
return 0;
}
};
static void verbose_error(bool verbose, int err, const char* fmt, ...) {
if (!verbose) return;
std::string msg = ErrorString(err);
if (fmt) {
msg += " at ";
va_list argp;
va_start(argp, fmt);
android::base::StringAppendV(&msg, fmt, argp);
va_end(argp);
}
sparse_print_verbose("%s\n", msg.c_str());
}
static int process_raw_chunk(struct sparse_file* s, unsigned int chunk_size,
SparseFileSource* source, unsigned int blocks, unsigned int block,
uint32_t* crc32) {
int ret;
int64_t len = blocks * s->block_size;
if (chunk_size % s->block_size != 0) {
return -EINVAL;
}
if (chunk_size / s->block_size != blocks) {
return -EINVAL;
}
ret = source->AddToSparseFile(s, len, block);
if (ret < 0) {
return ret;
}
if (crc32) {
ret = source->GetCrc32(crc32, len);
if (ret < 0) {
return ret;
}
} else {
source->Seek(len);
}
return 0;
}
static int process_fill_chunk(struct sparse_file* s, unsigned int chunk_size,
SparseFileSource* source, unsigned int blocks, unsigned int block,
uint32_t* crc32) {
int ret;
int chunk;
int64_t len = (int64_t)blocks * s->block_size;
uint32_t fill_val;
uint32_t* fillbuf;
unsigned int i;
if (chunk_size != sizeof(fill_val)) {
return -EINVAL;
}
ret = source->ReadValue(&fill_val, sizeof(fill_val));
if (ret < 0) {
return ret;
}
ret = sparse_file_add_fill(s, fill_val, len, block);
if (ret < 0) {
return ret;
}
if (crc32) {
/* Fill copy_buf with the fill value */
fillbuf = (uint32_t*)copybuf;
for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
fillbuf[i] = fill_val;
}
while (len) {
chunk = std::min(len, COPY_BUF_SIZE);
*crc32 = sparse_crc32(*crc32, copybuf, chunk);
len -= chunk;
}
}
return 0;
}
static int process_skip_chunk(struct sparse_file* s, unsigned int chunk_size,
SparseFileSource* source __unused, unsigned int blocks,
unsigned int block __unused, uint32_t* crc32) {
if (chunk_size != 0) {
return -EINVAL;
}
if (crc32) {
int64_t len = (int64_t)blocks * s->block_size;
memset(copybuf, 0, COPY_BUF_SIZE);
while (len) {
int chunk = std::min(len, COPY_BUF_SIZE);
*crc32 = sparse_crc32(*crc32, copybuf, chunk);
len -= chunk;
}
}
return 0;
}
static int process_crc32_chunk(SparseFileSource* source, unsigned int chunk_size, uint32_t* crc32) {
uint32_t file_crc32;
if (chunk_size != sizeof(file_crc32)) {
return -EINVAL;
}
int ret = source->ReadValue(&file_crc32, sizeof(file_crc32));
if (ret < 0) {
return ret;
}
if (crc32 != nullptr && file_crc32 != *crc32) {
return -EINVAL;
}
return 0;
}
static int process_chunk(struct sparse_file* s, SparseFileSource* source, unsigned int chunk_hdr_sz,
chunk_header_t* chunk_header, unsigned int cur_block, uint32_t* crc_ptr) {
int ret;
unsigned int chunk_data_size;
int64_t offset = source->GetOffset();
chunk_data_size = chunk_header->total_sz - chunk_hdr_sz;
switch (chunk_header->chunk_type) {
case CHUNK_TYPE_RAW:
ret =
process_raw_chunk(s, chunk_data_size, source, chunk_header->chunk_sz, cur_block, crc_ptr);
if (ret < 0) {
verbose_error(s->verbose, ret, "data block at %" PRId64, offset);
return ret;
}
return chunk_header->chunk_sz;
case CHUNK_TYPE_FILL:
ret = process_fill_chunk(s, chunk_data_size, source, chunk_header->chunk_sz, cur_block,
crc_ptr);
if (ret < 0) {
verbose_error(s->verbose, ret, "fill block at %" PRId64, offset);
return ret;
}
return chunk_header->chunk_sz;
case CHUNK_TYPE_DONT_CARE:
ret = process_skip_chunk(s, chunk_data_size, source, chunk_header->chunk_sz, cur_block,
crc_ptr);
if (chunk_data_size != 0) {
if (ret < 0) {
verbose_error(s->verbose, ret, "skip block at %" PRId64, offset);
return ret;
}
}
return chunk_header->chunk_sz;
case CHUNK_TYPE_CRC32:
ret = process_crc32_chunk(source, chunk_data_size, crc_ptr);
if (ret < 0) {
verbose_error(s->verbose, -EINVAL, "crc block at %" PRId64, offset);
return ret;
}
return 0;
default:
verbose_error(s->verbose, -EINVAL, "unknown block %04X at %" PRId64, chunk_header->chunk_type,
offset);
}
return 0;
}
static int sparse_file_read_sparse(struct sparse_file* s, SparseFileSource* source, bool crc) {
int ret;
unsigned int i;
sparse_header_t sparse_header;
chunk_header_t chunk_header;
uint32_t crc32 = 0;
uint32_t* crc_ptr = nullptr;
unsigned int cur_block = 0;
if (!copybuf) {
copybuf = (char*)malloc(COPY_BUF_SIZE);
}
if (!copybuf) {
return -ENOMEM;
}
if (crc) {
crc_ptr = &crc32;
}
ret = source->ReadValue(&sparse_header, sizeof(sparse_header));
if (ret < 0) {
return ret;
}
if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
return -EINVAL;
}
if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
return -EINVAL;
}
if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
return -EINVAL;
}
if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) {
return -EINVAL;
}
if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
/* Skip the remaining bytes in a header that is longer than
* we expected.
*/
source->Seek(sparse_header.file_hdr_sz - SPARSE_HEADER_LEN);
}
for (i = 0; i < sparse_header.total_chunks; i++) {
ret = source->ReadValue(&chunk_header, sizeof(chunk_header));
if (ret < 0) {
return ret;
}
if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
/* Skip the remaining bytes in a header that is longer than
* we expected.
*/
source->Seek(sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN);
}
ret = process_chunk(s, source, sparse_header.chunk_hdr_sz, &chunk_header, cur_block, crc_ptr);
if (ret < 0) {
return ret;
}
cur_block += ret;
}
if (sparse_header.total_blks != cur_block) {
return -EINVAL;
}
return 0;
}
static int sparse_file_read_normal(struct sparse_file* s, int fd) {
int ret;
uint32_t* buf = (uint32_t*)malloc(s->block_size);
unsigned int block = 0;
int64_t remain = s->len;
int64_t offset = 0;
unsigned int to_read;
unsigned int i;
bool sparse_block;
if (!buf) {
return -ENOMEM;
}
while (remain > 0) {
to_read = std::min(remain, (int64_t)(s->block_size));
ret = read_all(fd, buf, to_read);
if (ret < 0) {
error("failed to read sparse file");
free(buf);
return ret;
}
if (to_read == s->block_size) {
sparse_block = true;
for (i = 1; i < s->block_size / sizeof(uint32_t); i++) {
if (buf[0] != buf[i]) {
sparse_block = false;
break;
}
}
} else {
sparse_block = false;
}
if (sparse_block) {
/* TODO: add flag to use skip instead of fill for buf[0] == 0 */
sparse_file_add_fill(s, buf[0], to_read, block);
} else {
sparse_file_add_fd(s, fd, offset, to_read, block);
}
remain -= to_read;
offset += to_read;
block++;
}
free(buf);
return 0;
}
int sparse_file_read(struct sparse_file* s, int fd, bool sparse, bool crc) {
if (crc && !sparse) {
return -EINVAL;
}
if (sparse) {
SparseFileFdSource source(fd);
return sparse_file_read_sparse(s, &source, crc);
} else {
return sparse_file_read_normal(s, fd);
}
}
int sparse_file_read_buf(struct sparse_file* s, char* buf, bool crc) {
SparseFileBufSource source(buf);
return sparse_file_read_sparse(s, &source, crc);
}
static struct sparse_file* sparse_file_import_source(SparseFileSource* source, bool verbose,
bool crc) {
int ret;
sparse_header_t sparse_header;
int64_t len;
struct sparse_file* s;
ret = source->ReadValue(&sparse_header, sizeof(sparse_header));
if (ret < 0) {
verbose_error(verbose, ret, "header");
return nullptr;
}
if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
verbose_error(verbose, -EINVAL, "header magic");
return nullptr;
}
if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
verbose_error(verbose, -EINVAL, "header major version");
return nullptr;
}
if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
return nullptr;
}
if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) {
return nullptr;
}
len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz;
s = sparse_file_new(sparse_header.blk_sz, len);
if (!s) {
verbose_error(verbose, -EINVAL, nullptr);
return nullptr;
}
ret = source->SetOffset(0);
if (ret < 0) {
verbose_error(verbose, ret, "seeking");
sparse_file_destroy(s);
return nullptr;
}
s->verbose = verbose;
ret = sparse_file_read_sparse(s, source, crc);
if (ret < 0) {
sparse_file_destroy(s);
return nullptr;
}
return s;
}
struct sparse_file* sparse_file_import(int fd, bool verbose, bool crc) {
SparseFileFdSource source(fd);
return sparse_file_import_source(&source, verbose, crc);
}
struct sparse_file* sparse_file_import_buf(char* buf, bool verbose, bool crc) {
SparseFileBufSource source(buf);
return sparse_file_import_source(&source, verbose, crc);
}
struct sparse_file* sparse_file_import_auto(int fd, bool crc, bool verbose) {
struct sparse_file* s;
int64_t len;
int ret;
s = sparse_file_import(fd, verbose, crc);
if (s) {
return s;
}
len = lseek64(fd, 0, SEEK_END);
if (len < 0) {
return nullptr;
}
lseek64(fd, 0, SEEK_SET);
s = sparse_file_new(4096, len);
if (!s) {
return nullptr;
}
ret = sparse_file_read_normal(s, fd);
if (ret < 0) {
sparse_file_destroy(s);
return nullptr;
}
return s;
}