diff --git a/.gitignore b/.gitignore index 6bad8bb..33c6e83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,6 @@ -/ext/*.o -/ext/*.bundle -/ext/*.so -/ext/*.dll -/ext/mkmf.log +/tmp /lib/simple_mmap/*.bundle /lib/simple_mmap/*.so /lib/simple_mmap/*.dll -/ext/Makefile /pkg /doc/* diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f939f55 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: ruby +rvm: + - 2.0.0 + - 1.9.3 + - 1.9.2 + - 1.8.7 diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..8e2ab11 --- /dev/null +++ b/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in simple_mmap.gemspec +gemspec diff --git a/Rakefile b/Rakefile index a2ceecc..a81bac4 100644 --- a/Rakefile +++ b/Rakefile @@ -1,38 +1,15 @@ -# encoding: utf-8 +require "bundler/gem_tasks" -require 'rubygems' -require 'hoe' -require './lib/simple_mmap/version.rb' -require "rake/clean" - -Hoe.new('simple_mmap', SimpleMmap::VERSION) do |p| - p.rubyforge_name = 'simple-mmap' - p.developer('Johan Sørensen', 'johan@johansorensen.com') - p.spec_extras = { - "extensions" => ["Rakefile"] - } -end - -DLEXT = Config::CONFIG['DLEXT'] - -file 'ext/Makefile' => FileList['ext/{*.c,*.h,*.rb}'] do - chdir('ext') { ruby 'extconf.rb' } +require 'rake/extensiontask' +Rake::ExtensionTask.new('simple_mmap') do |ext| + ext.name = "mapped_file" + ext.lib_dir = File.join('lib', 'simple_mmap') end -CLEAN.include 'ext/Makefile', 'ext/mkmf.log' -file "ext/mapped_file.#{DLEXT}" => FileList['ext/Makefile', 'ext/*.{c,h,rb}'] do |f| - sh 'cd ext && make' +require 'rake/testtask' +Rake::TestTask.new(:test => :compile) do |test| + test.libs << 'ext' + test.verbose = true end -CLEAN.include 'ext/*.{o,bundle,so,dll}' - -file "lib/simple_mmap/mapped_file.#{DLEXT}" => "ext/mapped_file.#{DLEXT}" do |f| - cp f.prerequisites, "lib/simple_mmap/", :preserve => true -end -#CLEAN.include "lib/simple_mmap/mapped_file.#{DLEXT}" - -desc 'Build the mapped_file extension' -task :build => "lib/simple_mmap/mapped_file.#{DLEXT}" - -task :test => [:build] task :default => :test diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..65087b4 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.1.4 diff --git a/ext/extconf.rb b/ext/extconf.rb deleted file mode 100644 index f4fe914..0000000 --- a/ext/extconf.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'mkmf' - -dir_config('mapped_file') -have_func('mmap') -create_makefile('mapped_file') diff --git a/ext/simple_mmap/extconf.rb b/ext/simple_mmap/extconf.rb new file mode 100644 index 0000000..c41cbc3 --- /dev/null +++ b/ext/simple_mmap/extconf.rb @@ -0,0 +1,4 @@ +require 'mkmf' + +have_func('mmap') +create_makefile('simple_mmap/mapped_file') diff --git a/ext/mapped_file.c b/ext/simple_mmap/mapped_file.c similarity index 61% rename from ext/mapped_file.c rename to ext/simple_mmap/mapped_file.c index d835ab5..c917c44 100644 --- a/ext/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -28,11 +28,31 @@ #include #include "ruby.h" +#ifndef NUM2SIZET +# include +# include +# if SIZE_MAX > ULONG_MAX +# define NUM2SIZET NUM2ULL +# define SIZET2NUM ULONG2NUM +# elif SIZE_MAX > UINT_MAX +# define NUM2SIZET NUM2ULONG +# define SIZET2NUM ULONG2NUM +# elif SIZE_MAX > USHRT_MAX +# define NUM2SIZET NUM2UINT +# define SIZET2NUM UINT2NUM +# else +# define NUM2SIZET NUM2USHORT +# define SIZET2NUM USHORT2NUM +# endif +#endif + // a memory mapped file typedef struct { - int fd; + int fd; caddr_t map; - size_t len; + size_t len; + caddr_t read_base; + size_t read_len; } simple_mmap_map; static VALUE mod_simple_mmap; @@ -45,7 +65,7 @@ static VALUE sm_map_data; * * mmap() the file at +path+ */ -static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) +static VALUE sm_mapped_file_initialize(int argc, VALUE *argv, VALUE vself) { int fd = -1; size_t length; @@ -54,6 +74,15 @@ static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) VALUE vsm_map; simple_mmap_map *sm_map; + VALUE filename, voffset, vlength; + off_t offset, offset_mod; + + rb_scan_args(argc, argv, "12", &filename, &voffset, &vlength); + if (!NIL_P(voffset) && NUM2LL(voffset) < 0) + rb_raise(rb_eRangeError, "offset out of range: %lld", NUM2LL(voffset)); + if (!NIL_P(vlength) && NUM2LL(vlength) < 0) + rb_raise(rb_eRangeError, "length out of range: %lld", NUM2LL(vlength)); + fd = open(RSTRING_PTR(filename), O_RDONLY); if (fd == -1) { rb_raise(rb_eArgError, "Failed to open file %s", RSTRING_PTR(filename)); @@ -65,10 +94,15 @@ static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) rb_raise(rb_eArgError, "Failed to stat file %s", RSTRING_PTR(filename)); close(fd); } - length = st.st_size; + offset = NIL_P(voffset) ? 0 : NUM2SIZET(voffset); + length = NIL_P(vlength) ? st.st_size : NUM2SIZET(vlength); + if (offset + length > st.st_size) length = st.st_size - offset; + offset_mod = offset % sysconf(_SC_PAGESIZE); + offset = offset - offset_mod; + length = length + offset_mod; // do the mmap - base = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); + base = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, offset); if (base == (caddr_t) -1) { rb_raise(rb_eArgError, "Failed to mmap file %s", RSTRING_PTR(filename)); close(fd); @@ -80,6 +114,8 @@ static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) sm_map->fd = fd; sm_map->map = base; sm_map->len = length; + sm_map->read_base = base + offset_mod; + sm_map->read_len = length - offset_mod; rb_ivar_set(vself, rb_intern("@mmap_data"), vsm_map); return Qnil; @@ -109,41 +145,50 @@ static VALUE sm_mapped_file_close(VALUE vself) } /* - * Document-method: close + * Document-method: read_window_data * call-seq: obj.read_window_data(offset, length) * * Read +length+ bytes starting at +offset+ */ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE vlength) { - size_t offset = NUM2INT(voffset); - size_t length = NUM2INT(vlength); - char buff[length]; + size_t offset, length; VALUE vsm_map; simple_mmap_map *sm_map; - sm_map = ALLOC(simple_mmap_map); + if (NUM2LL(voffset) < 0) return Qnil; + if (NUM2LL(vlength) < 0) rb_raise(rb_eRangeError, "length out of range: %lld", NUM2LL(vlength)); + offset = NUM2SIZET(voffset); + length = NUM2SIZET(vlength); + vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); - if (offset > sm_map->len) { - return Qnil; - } - - size_t curr; - curr = offset; - size_t i; - for(i = 0; i < length; ++i) { - //printf("i=%i offset=%i length=%i curr=%i map->len=%i\n", i, offset, length, curr, sm_map->len); - if ((curr + i) > sm_map->len) - break; - buff[i] = sm_map->map[curr++]; + if (offset > sm_map->read_len) return Qnil; + + // If the range overflows, return part that overlaps + if ((offset + length) > sm_map->read_len) { + length = sm_map->read_len - offset; } - - if ((offset + length) > sm_map->len) - return rb_str_new(buff, ((offset+length) - sm_map->len)-1); - - return rb_str_new(buff, length); + + return rb_str_new(sm_map->read_base + offset, length); +} + +/* + * Document-method: size + * call-seq: obj.size + * + * Return size of mapped file + */ + +static VALUE sm_mapped_file_size(VALUE vself) +{ + VALUE vsm_map; + simple_mmap_map *sm_map; + + vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); + Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); + return SIZET2NUM(sm_map->read_len); } void Init_mapped_file() @@ -151,9 +196,10 @@ void Init_mapped_file() mod_simple_mmap = rb_define_module("SimpleMmap"); sm_mapped_file = rb_define_class_under(mod_simple_mmap, "MappedFile", rb_cObject); - rb_define_private_method(sm_mapped_file, "initialize", sm_mapped_file_initialize, 1); + rb_define_private_method(sm_mapped_file, "initialize", sm_mapped_file_initialize, -1); rb_define_method(sm_mapped_file, "close", sm_mapped_file_close, 0); rb_define_method(sm_mapped_file, "read_window_data", sm_mapped_file_read_window_data, 2); + rb_define_method(sm_mapped_file, "size", sm_mapped_file_size, 0); sm_map_data = rb_define_class_under(sm_mapped_file, "MmapData", rb_cObject); } diff --git a/lib/simple_mmap.rb b/lib/simple_mmap.rb index 972ea91..cfbde3e 100644 --- a/lib/simple_mmap.rb +++ b/lib/simple_mmap.rb @@ -1,4 +1,4 @@ -# Copyright (c) 2009 Johan Sørensen +# Copyright (c) 2009-2016 Johan Sørensen # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the diff --git a/lib/simple_mmap/file_window.rb b/lib/simple_mmap/file_window.rb index b6050f1..b6caa4b 100644 --- a/lib/simple_mmap/file_window.rb +++ b/lib/simple_mmap/file_window.rb @@ -25,10 +25,10 @@ module SimpleMmap class FileWindow # Create a mmap'ed window for file at +path+ # You are responsible for closing it when you're done using #close - def initialize(path) + def initialize(path, offset = nil, length = nil) @path = path @offset = 0 - @mmap = SimpleMmap::MappedFile.new(@path) + @mmap = SimpleMmap::MappedFile.new(@path, offset, length) end attr_reader :path, :offset @@ -70,11 +70,11 @@ def [](*index) offset = index length = 0 when Range - offset = index.begin - length = index.end - index.begin - unless index.exclude_end? - length += 1 - end + offset = index.begin < 0 ? index.begin + @mmap.size : index.begin + return nil if offset < 0 or offset > @mmap.size + length = (index.end < 0 ? index.end + @mmap.size : index.end) - offset + length += 1 unless index.exclude_end? + return '' if length <= 0 end @offset = offset + length @@ -92,5 +92,10 @@ def read(length) @offset += length data end + + # Return size of mapped file + def size + @mmap.size + end end end diff --git a/lib/simple_mmap/version.rb b/lib/simple_mmap/version.rb index 539e453..ec55846 100644 --- a/lib/simple_mmap/version.rb +++ b/lib/simple_mmap/version.rb @@ -1,24 +1,3 @@ -# Copyright (c) 2009 Johan Sørensen -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# 'Software'), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - module SimpleMmap - VERSION = '1.0.0' + VERSION = "1.2" end diff --git a/simple-mmap.gemspec b/simple-mmap.gemspec new file mode 100644 index 0000000..9d067ea --- /dev/null +++ b/simple-mmap.gemspec @@ -0,0 +1,26 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'simple_mmap/version' + +Gem::Specification.new do |spec| + spec.name = "simple-mmap" + spec.version = SimpleMmap::VERSION + spec.authors = ["Johan Sørensen", "Pieter Noordhuis", "TAKADA Daisuke"] + spec.email = ["pcnoordhuis@gmail.com", "shoulderpower@gmail.com"] +# spec.description = %q{TODO: Write a gem description} + spec.description = %q{A simplistic interface for reading memory mapped files} + spec.summary = %q{A simplistic interface for reading memory mapped files} + spec.homepage = "http://github.com/pietern/simple-mmap" + spec.license = "MIT" + + spec.files = `git ls-files`.split($/) + spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } + spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) + spec.require_paths = ["ext", "lib"] + spec.extensions << 'ext/simple_mmap/extconf.rb' + + spec.add_development_dependency "bundler", "~> 1.3" + spec.add_development_dependency "rake" + spec.add_development_dependency 'rake-compiler', '>= 0.7.0' +end diff --git a/test/test_file_window.rb b/test/test_file_window.rb index 77d18a4..0ae42e5 100644 --- a/test/test_file_window.rb +++ b/test/test_file_window.rb @@ -5,7 +5,7 @@ class TestFileWindow < Test::Unit::TestCase def setup @file = Tempfile.new("TestFileWindow.data") - File.open(@file.path, "w"){|f| f.puts(('a'..'z').to_a.join) } + File.open(@file.path, "w"){|f| f.write(('a'..'z').to_a.join) } @fw = SimpleMmap::FileWindow.new(@file.path) end @@ -39,6 +39,19 @@ def test_get_single_byte_at_current_offset_with_index assert_equal 1, @fw.offset end + def test_get_bytes_past_length + assert_equal "z", @fw[25, 10] + end + + def test_get_all_bytes + assert_equal ('a'..'z').to_a.join, @fw[0, 26] + end + + def test_nil_on_negative_index + assert_equal nil, @fw[-1] + assert_equal nil, @fw[-1, 2] + end + def test_get_from_x_to_y_with_index_comma_notation assert_equal "cde", @fw[2, 3] assert_equal 5, @fw.pos @@ -59,4 +72,8 @@ def test_read assert_equal "abc", @fw.read(3) assert_equal 3, @fw.offset end + + def test_size + assert_equal 26, @fw.size + end end