dan/parsley
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding USDA weights; usda_importer refactor

Browse Source
This commit is contained in:
Dan Elbert 2016-03-09 18:53:47 -06:00
parent 068b01a7c8
commit 2f752eae61
10 changed files with 15727 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/models/usda_food.rb
View File

@ -1,6 +1,8 @@
class UsdaFood < ActiveRecord::Base
include TokenizedLike
has_many :usda_food_weights
def self.search(query)
tokens = query.to_s.split(' ')

5
app/models/usda_food_weight.rb Normal file
View File

@ -0,0 +1,5 @@
class UsdaFoodWeight < ActiveRecord::Base
belongs_to :usda_food
end

13
db/migrate/20160309182253_create_usda_food_weights.rb Normal file
View File

@ -0,0 +1,13 @@
class CreateUsdaFoodWeights < ActiveRecord::Migration
def change
create_table :usda_food_weights do |t|
t.integer :usda_food_id, index: true, null: false
t.decimal :amount, precision: 5, scale: 3
t.string :description
t.decimal :gram_weight, precision: 7, scale: 1
t.timestamps null: false
end
end
end

13
db/schema.rb
View File

@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20160303180854) do
ActiveRecord::Schema.define(version: 20160309182253) do
create_table "ingredients", force: :cascade do |t|
t.string "name"
@ -68,6 +68,17 @@ ActiveRecord::Schema.define(version: 20160303180854) do
t.integer "user_id"
end
create_table "usda_food_weights", force: :cascade do |t|
t.integer "usda_food_id", null: false
t.decimal "amount", precision: 5, scale: 3
t.string "description"
t.decimal "gram_weight", precision: 7, scale: 1
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
add_index "usda_food_weights", ["usda_food_id"], name: "index_usda_food_weights_on_usda_food_id"
create_table "usda_foods", force: :cascade do |t|
t.string "ndbn", limit: 5, null: false
t.string "long_description"

355
lib/usda_importer.rb
View File

@ -2,101 +2,126 @@ require 'csv'
class UsdaImporter
ABBREV_COLUMNS = [
'NDB_No',
'Shrt_Desc',
'Water',
'Energ_Kcal',
'Protein',
'Lipid_Tot',
'Ash',
'Carbohydrt',
'Fiber_TD',
'Sugar_Tot',
'Calcium',
'Iron',
'Magnesium',
'Phosphorus',
'Potassium',
'Sodium',
'Zinc',
'Copper',
'Manganese',
'Selenium',
'Vit_C',
'Thiamin',
'Riboflavin',
'Niacin',
'Panto_acid',
'Vit_B6',
'Folate_Tot',
'Folic_acid',
'Food_Folate',
'Folate_DFE',
'Choline_Tot',
'Vit_B12',
'Vit_A_IU',
'Vit_A_RAE',
'Retinol',
'Alpha_Carot',
'Beta_Carot',
'Beta_Crypt',
'Lycopene',
'Lut+Zea',
'Vit_E',
'Vit_D_mcg',
'Vit_D_IU',
'Vit_K',
'FA_Sat',
'FA_Mono',
'FA_Poly',
'Cholestrl',
'GmWt_1',
'GmWt_Desc1',
'GmWt_2',
'GmWt_Desc2',
'Refuse_Pct'
]
FILES = {
abbreviated: {
filename: 'ABBREV.txt',
columns: [
'NDB_No',
'Shrt_Desc',
'Water',
'Energ_Kcal',
'Protein',
'Lipid_Tot',
'Ash',
'Carbohydrt',
'Fiber_TD',
'Sugar_Tot',
'Calcium',
'Iron',
'Magnesium',
'Phosphorus',
'Potassium',
'Sodium',
'Zinc',
'Copper',
'Manganese',
'Selenium',
'Vit_C',
'Thiamin',
'Riboflavin',
'Niacin',
'Panto_acid',
'Vit_B6',
'Folate_Tot',
'Folic_acid',
'Food_Folate',
'Folate_DFE',
'Choline_Tot',
'Vit_B12',
'Vit_A_IU',
'Vit_A_RAE',
'Retinol',
'Alpha_Carot',
'Beta_Carot',
'Beta_Crypt',
'Lycopene',
'Lut+Zea',
'Vit_E',
'Vit_D_mcg',
'Vit_D_IU',
'Vit_K',
'FA_Sat',
'FA_Mono',
'FA_Poly',
'Cholestrl',
'GmWt_1',
'GmWt_Desc1',
'GmWt_2',
'GmWt_Desc2',
'Refuse_Pct'
],
map: {
ndbn: 'NDB_No',
short_description: 'Shrt_Desc',
water: 'Water',
kcal: 'Energ_Kcal',
protein: 'Protein',
lipid: 'Lipid_Tot',
ash: 'Ash',
carbohydrates: 'Carbohydrt',
fiber: 'Fiber_TD',
sugar: 'Sugar_Tot',
gram_weight_1: 'GmWt_1',
gram_weight_2: 'GmWt_2',
gram_weight_desc_1: 'GmWt_Desc1',
gram_weight_desc_2: 'GmWt_Desc2',
refuse_percent: 'Refuse_Pct'
}
},
FOOD_DATA_COLUMNS = [
'NDB_No',
'FdGrp_Cd',
'Long_Desc',
'Shrt_Desc',
'ComName',
'ManufacName',
'Survey',
'Ref_desc',
'Refuse',
'SciName',
'N_Factor',
'Pro_Factor',
'Fat_Factor',
'CHO_Factor'
]
food_data: {
filename: 'FOOD_DES.txt',
columns: [
'NDB_No',
'FdGrp_Cd',
'Long_Desc',
'Shrt_Desc',
'ComName',
'ManufacName',
'Survey',
'Ref_desc',
'Refuse',
'SciName',
'N_Factor',
'Pro_Factor',
'Fat_Factor',
'CHO_Factor'
],
map: {
scientific_name: 'SciName',
refuse_description: 'Ref_desc',
long_description: 'Long_Desc'
}
},
ABBREV_COLUMN_MAP = {
ndbn: 'NDB_No',
short_description: 'Shrt_Desc',
water: 'Water',
kcal: 'Energ_Kcal',
protein: 'Protein',
lipid: 'Lipid_Tot',
ash: 'Ash',
carbohydrates: 'Carbohydrt',
fiber: 'Fiber_TD',
sugar: 'Sugar_Tot',
gram_weight_1: 'GmWt_1',
gram_weight_2: 'GmWt_2',
gram_weight_desc_1: 'GmWt_Desc1',
gram_weight_desc_2: 'GmWt_Desc2',
refuse_percent: 'Refuse_Pct'
}
FOOD_DATA_COLUMN_MAP = {
scientific_name: 'SciName',
refuse_description: 'Ref_desc',
long_description: 'Long_Desc'
weights: {
filename: 'WEIGHT.txt',
map_into: 'usda_food_weights',
columns: [
'NDB_No',
'Seq',
'Amount',
'Msre_Desc',
'Gm_Wgt',
'Num_Data_Pts',
'Std_Dev'
],
map: {
amount: 'Amount',
description: 'Msre_Desc',
gram_weight: 'Gm_Wgt'
}
}
}
def initialize(directory)
@ -105,47 +130,145 @@ class UsdaImporter
def import
UsdaFoodWeight.delete_all
UsdaFood.delete_all
food_data_lookup = {}
sorted_files = {}
opened_files = {}
CSV.open(File.join(@directory, 'FOOD_DES.txt'), 'r:iso-8859-1:utf-8', csv_options(FOOD_DATA_COLUMNS)) do |csv|
csv.each do |row|
food_data_lookup[row['NDB_No']] = row.to_h
# Sort each file by NDB_No
FILES.each do |name, data|
filename = File.join(@directory, data[:filename])
sorted_filename = "#{filename}.sorted"
idx = data[:columns].index('NDB_No')
if idx
idx += 1
`sort -n -t'^' -k#{idx}.2,#{idx}.6 #{filename} > #{sorted_filename}`
end
sorted_files[name] = sorted_filename
end
CSV.open(File.join(@directory, 'ABBREV.txt'), 'r:iso-8859-1:utf-8', csv_options(ABBREV_COLUMNS)) do |csv|
csv.each_slice(500) do |slice|
begin
sorted_files.each do |name, filename|
data = FILES[name]
opened_files[name] = CSV.open(filename, 'r:iso-8859-1:utf-8', csv_options(data[:columns]))
end
build_enumerator(opened_files).each_slice(500) do |slice|
UsdaFood.transaction do
slice.each do |data|
attributes = slice.map do |row|
attrs = Hash[ABBREV_COLUMN_MAP.map { |db, col| [db, row[col]] }]
lookup = food_data_lookup[attrs[:ndbn]]
if lookup
extra_attrs = Hash[FOOD_DATA_COLUMN_MAP.map { |db, col| [db, lookup[col]] }]
attrs.merge!(extra_attrs)
food = UsdaFood.new
data.each do |name, rows|
file_info = FILES[name]
obj = food
if file_info[:map_into]
obj = food.send(file_info[:map_into]).build
end
rows.each do |row|
file_info[:map].each do |db, col|
obj.send("#{db}=", row[col])
end
end
end
attrs
food.save!
end
UsdaFood.create(attributes)
end
end
ensure
opened_files.each { |k, v| v.close }
sorted_files.each { |k, v| `rm #{v}` }
end
usda_items = Hash[UsdaFood.where(ndbn: Ingredient.select(:ndbn)).map { |uf| [uf.ndbn, uf] }]
# UsdaFood.delete_all
#
# food_data_lookup = {}
#
# CSV.open(File.join(@directory, 'FOOD_DES.txt'), 'r:iso-8859-1:utf-8', csv_options(FOOD_DATA_COLUMNS)) do |csv|
# csv.each do |row|
# food_data_lookup[row['NDB_No']] = row.to_h
# end
# end
#
# CSV.open(File.join(@directory, 'ABBREV.txt'), 'r:iso-8859-1:utf-8', csv_options(ABBREV_COLUMNS)) do |csv|
# csv.each_slice(500) do |slice|
# UsdaFood.transaction do
#
# attributes = slice.map do |row|
# attrs = Hash[ABBREV_COLUMN_MAP.map { |db, col| [db, row[col]] }]
# lookup = food_data_lookup[attrs[:ndbn]]
# if lookup
# extra_attrs = Hash[FOOD_DATA_COLUMN_MAP.map { |db, col| [db, lookup[col]] }]
# attrs.merge!(extra_attrs)
# end
# attrs
# end
#
# UsdaFood.create(attributes)
#
# end
# end
# end
#
# usda_items = Hash[UsdaFood.where(ndbn: Ingredient.select(:ndbn)).map { |uf| [uf.ndbn, uf] }]
#
# Ingredient.where('ndbn IS NOT NULL').each do |i|
# item = usda_items[i.ndbn]
#
# if item
# i.set_usda_food(item)
# i.save
# end
# end
Ingredient.where('ndbn IS NOT NULL').each do |i|
item = usda_items[i.ndbn]
end
def build_enumerator(opened_files)
enumerate_data = {}
opened_files.each do |name, csv|
csv_enumerator = csv.each
enumerate_data[name] = {
enumerator: csv_enumerator,
done: false,
next_ndbn: csv_enumerator.peek['NDB_No']
}
end
Enumerator.new do |yielder|
loop do
break if enumerate_data.values.all? { |d| d[:done] }
current_ndbn = enumerate_data.values.map { |d| d[:next_ndbn] }.min
results = Hash.new { |hash, key| hash[key] = [] }
enumerate_data.each do |name, data|
unless data[:done]
begin
while data[:next_ndbn] == current_ndbn
results[name] << data[:enumerator].next
data[:next_ndbn] = data[:enumerator].peek['NDB_No']
end
rescue StopIteration
data[:done] = true
end
end
end
yielder << results
if item
i.set_usda_food(item)
i.save
end
end
end
end
def csv_options(headers)

5
spec/lib/usda_importer_spec.rb Normal file
View File

@ -0,0 +1,5 @@
require 'rails_helper'
RSpec.describe UsdaImporter do
end

2
spec/test_data/ABBREV.txt Normal file
View File

@ -0,0 +1,2 @@
~01001~^~BUTTER,WITH SALT~^15.87^717^0.85^81.11^2.11^0.06^0.0^0.06^24^0.02^2^24^24^643^0.09^0.000^0.000^1.0^0.0^0.005^0.034^0.042^0.110^0.003^3^0^3^3^18.8^0.17^2499^684^671^0^158^0^0^0^2.32^0.0^0^7.0^51.368^21.021^3.043^215^5.0^~1 pat, (1" sq, 1/3" high)~^14.2^~1 tbsp~^0
~01006~^~CHEESE,BRIE~^48.42^334^20.75^27.68^2.70^0.45^0.0^0.45^184^0.50^20^188^152^629^2.38^0.019^0.034^14.5^0.0^0.070^0.520^0.380^0.690^0.235^65^0^65^65^15.4^1.65^592^174^173^0^9^0^0^0^0.24^0.5^20^2.3^17.410^8.013^0.826^100^28.35^~1 oz~^144^~1 cup, sliced~^0

2
spec/test_data/FOOD_DES.txt Normal file
View File

@ -0,0 +1,2 @@
~01001~^~0100~^~Butter, salted~^~BUTTER,WITH SALT~^~~^~~^~Y~^~~^0^~~^6.38^4.27^8.79^3.87
~01006~^~0100~^~Cheese, brie~^~CHEESE,BRIE~^~~^~~^~Y~^~~^0^~~^6.38^4.27^8.79^3.87

9
spec/test_data/WEIGHT.txt Normal file
View File

@ -0,0 +1,9 @@
~01001~^1^1^~pat (1" sq, 1/3" high)~^5.0^^
~01001~^2^1^~tbsp~^14.2^^
~01001~^3^1^~cup~^227^^
~01001~^4^1^~stick~^113^^
~01006~^1^1^~oz~^28.35^^
~01006~^2^1^~cup, sliced~^144^^
~01006~^3^1^~cup, melted~^240^^
~01006~^4^1^~cubic inch~^17^^
~01006~^5^1^~package (4.5 oz)~^128^^

15438
vendor/data/usda/WEIGHT.txt vendored Normal file
View File

File diff suppressed because it is too large Load Diff