Skip to content

Commit

Permalink
Fix for #1: Fido Positions format has changed
Browse files Browse the repository at this point in the history
  • Loading branch information
Reed Es committed Jul 30, 2021
1 parent f592f6b commit 97c8a23
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 23 deletions.
14 changes: 10 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,20 @@ $ finport detect mystery.txt

### Fido (Fidelity) Positions

To transform the "Portfolio_Positions_Mmm-dd-yyyy.csv" export requires two commands, as there are two outputs, account holdings and securities:
To transform the "Portfolio_Positions_Mmm-dd-yyyy.csv" export requires three(3) commands, as there are three outputs: accounts, account holdings, and securities:

```bash
$ finport transform Portfolio_Positions_Jun-30-2021.csv --output-schema openalloc/account
$ finport transform Portfolio_Positions_Jun-30-2021.csv --output-schema openalloc/holding
$ finport transform Portfolio_Positions_Jun-30-2021.csv --output-schema openalloc/security
```

Each command above will produce comma-separated value data in the following schemas, respectively.

Output schemas: [openalloc/holding](https://github.com/openalloc/AllocData#mholding) and [openalloc/security](https://github.com/openalloc/AllocData#msecurity)
Output schemas:
* [openalloc/account](https://github.com/openalloc/AllocData#maccount)
* [openalloc/holding](https://github.com/openalloc/AllocData#mholding)
* [openalloc/security](https://github.com/openalloc/AllocData#msecurity)

### Fido (Fidelity) Purchases

Expand All @@ -72,7 +76,8 @@ $ finport transform Realized_Gain_Loss_Account_00000000.csv

The command above will produce comma-separated value data in the following schema.

Output schema: [openalloc/history](https://github.com/openalloc/AllocData#mhistory)
Output schema:
* [openalloc/history](https://github.com/openalloc/AllocData#mhistory)

### AllocSmart (Allocate Smartly) Export

Expand All @@ -84,7 +89,8 @@ $ finport transform "Allocate Smartly Model Portfolio.csv"

The command above will produce comma-separated value data in the following schema.

Output schema: [openalloc/allocation](https://github.com/openalloc/AllocData#mallocation)
Output schema:
* [openalloc/allocation](https://github.com/openalloc/AllocData#mallocation)

## Command Line

Expand Down
2 changes: 1 addition & 1 deletion Sources/Importers/AllocSmart.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class AllocSmart: FINporter {
"US Total Market": .total,
]

override var name: String { "AssetValue Smart" }
override var name: String { "Alloc Smart" }
override var id: String { "alloc_smart" }
override var description: String { "Detect and decode export files from Allocate Smartly." }
override var sourceFormats: [AllocFormat] { [.CSV] }
Expand Down
25 changes: 21 additions & 4 deletions Sources/Importers/FidoPositions.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@ class FidoPositions: FINporter {
override var id: String { "fido_positions" }
override var description: String { "Detect and decode position export files from Fidelity." }
override var sourceFormats: [AllocFormat] { [.CSV] }
override var outputSchemas: [AllocSchema] { [.allocHolding, .allocSecurity] }
override var outputSchemas: [AllocSchema] { [.allocAccount, .allocHolding, .allocSecurity] }

private let trimFromTicker = CharacterSet(charactersIn: "*")

override func detect(dataPrefix: Data) throws -> DetectResult {
let headerRE = #"""
Account Name/Number,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
Account Number,Account Name,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
"""#

guard let str = String(data: dataPrefix, encoding: .utf8),
Expand Down Expand Up @@ -70,7 +70,7 @@ class FidoPositions: FINporter {
var items = [T.Row]()

// should match all lines, until a blank line or end of block/file
let csvRE = #"Account Name/Number,Symbol,Description,Quantity,(?:.+(\r?\n|\Z))+"#
let csvRE = #"Account Number,Account Name,Symbol,Description,Quantity,(?:.+(\r?\n|\Z))+"#

if let csvRange = str.range(of: csvRE, options: .regularExpression) {
let csvStr = str[csvRange]
Expand All @@ -79,6 +79,8 @@ class FidoPositions: FINporter {
var item: T.Row?

switch outputSchema_ {
case .allocAccount:
item = account(row, rejectedRows: &rejectedRows)
case .allocHolding:
item = holding(row, rejectedRows: &rejectedRows)
case .allocSecurity:
Expand All @@ -98,7 +100,7 @@ class FidoPositions: FINporter {

private func holding(_ row: [String: String], rejectedRows: inout [AllocBase.Row]) -> AllocBase.Row? {
// required values
guard let accountID = MHolding.parseString(row["Account Name/Number"]),
guard let accountID = MHolding.parseString(row["Account Number"]),
accountID.count > 0,
let securityID = MHolding.parseString(row["Symbol"], trimCharacters: trimFromTicker),
securityID.count > 0,
Expand Down Expand Up @@ -140,4 +142,19 @@ class FidoPositions: FINporter {
MSecurity.CodingKeys.updatedAt.rawValue: timestamp
]
}

private func account(_ row: [String: String], rejectedRows: inout [AllocBase.Row]) -> AllocBase.Row? {
guard let accountID = MHolding.parseString(row["Account Number"]),
accountID.count > 0,
let title = MHolding.parseString(row["Account Name"])
else {
rejectedRows.append(row)
return nil
}

return [
MAccount.CodingKeys.accountID.rawValue: accountID,
MAccount.CodingKeys.title.rawValue: title
]
}
}
38 changes: 24 additions & 14 deletions Tests/Importers/FidoPositionsTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,14 @@ final class FidoPositionsTests: XCTestCase {
}

func testTargetSchema() {
let expected: [AllocSchema] = [.allocHolding, .allocSecurity]
let expected: [AllocSchema] = [.allocAccount, .allocHolding, .allocSecurity]
let actual = imp.outputSchemas
XCTAssertEqual(expected, actual)
}

func testDetectFailsDueToHeaderMismatch() throws {
let badHeader = """
AXXount Name/Number,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
AXXount Number,Account Name,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
"""
let expected: FINporter.DetectResult = [:]
let actual = try imp.detect(dataPrefix: badHeader.data(using: .utf8)!)
Expand All @@ -50,18 +50,18 @@ final class FidoPositionsTests: XCTestCase {

func testDetectSucceeds() throws {
let header = """
Account Name/Number,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
Account Number,Account Name,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
"""
let expected: FINporter.DetectResult = [.allocHolding: [.CSV], .allocSecurity: [.CSV]]
let expected: FINporter.DetectResult = [.allocAccount: [.CSV], .allocHolding: [.CSV], .allocSecurity: [.CSV]]
let actual = try imp.detect(dataPrefix: header.data(using: .utf8)!)
XCTAssertEqual(expected, actual)
}

func testDetectViaMain() throws {
let header = """
Account Name/Number,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
Account Number,Account Name,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
"""
let expected: FINporter.DetectResult = [.allocHolding: [.CSV], .allocSecurity: [.CSV]]
let expected: FINporter.DetectResult = [.allocAccount: [.CSV], .allocHolding: [.CSV], .allocSecurity: [.CSV]]
let main = FINprospector()
let data = header.data(using: .utf8)!
let actual = try main.prospect(sourceFormats: [.CSV], dataPrefix: data)
Expand All @@ -74,22 +74,22 @@ final class FidoPositionsTests: XCTestCase {

func testParse() throws {
for str in [
"Account Name/Number,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today\'s Gain/Loss Dollar,Today\'s Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type\r\nZ00000000,VWO,VANGUARD INTL EQUITY INDEX FDS FTSE EMR MKT ETF,900,$50.922,+$0.160,\"$45,900.35\",+$150.25,+0.32%,\"+$11,945.20\",+31.10%,15.05%,\"$38,362.05\",$28.96,Cash,\r\nZ00000001,VOO,VANGUARD S&P 500 ETF,800,$40.922,+$0.160,\"$45,900.35\",+$150.25,+0.32%,\"+$11,945.20\",+31.10%,15.05%,\"$38,362.05\",$18.96,Cash,\r\n\r\nXXX",
"Account Number,Account Name,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today\'s Gain/Loss Dollar,Today\'s Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type\r\nZ00000000,AAAA,VWO,VANGUARD INTL EQUITY INDEX FDS FTSE EMR MKT ETF,900,$50.922,+$0.160,\"$45,900.35\",+$150.25,+0.32%,\"+$11,945.20\",+31.10%,15.05%,\"$38,362.05\",$28.96,Cash,\r\nZ00000001,BBBB,VOO,VANGUARD S&P 500 ETF,800,$40.922,+$0.160,\"$45,900.35\",+$150.25,+0.32%,\"+$11,945.20\",+31.10%,15.05%,\"$38,362.05\",$18.96,Cash,\r\n\r\nXXX",

// testParseWithLFToFirstBlankLine
"""
Account Name/Number,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
Z00000000,VWO,VANGUARD INTL EQUITY INDEX FDS FTSE EMR MKT ETF,900,$50.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$28.96,Cash,
Z00000001,VOO,VANGUARD S&P 500 ETF,800,$40.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$18.96,Cash,
Account Number,Account Name,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
Z00000000,AAAA,VWO,VANGUARD INTL EQUITY INDEX FDS FTSE EMR MKT ETF,900,$50.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$28.96,Cash,
Z00000001,BBBB,VOO,VANGUARD S&P 500 ETF,800,$40.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$18.96,Cash,
XXX
""",

// testParseWithLFToFirstBlankLine
"""
Account Name/Number,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
Z00000000,VWO,VANGUARD INTL EQUITY INDEX FDS FTSE EMR MKT ETF,900,$50.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$28.96,Cash,
Z00000001,VOO,VANGUARD S&P 500 ETF,800,$40.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$18.96,Cash,
Account Number,Account Name,Symbol,Description,Quantity,Last Price,Last Price Change,Current Value,Today's Gain/Loss Dollar,Today's Gain/Loss Percent,Total Gain/Loss Dollar,Total Gain/Loss Percent,Percent Of Account,Cost Basis,Cost Basis Per Share,Type
Z00000000,AAAA,VWO,VANGUARD INTL EQUITY INDEX FDS FTSE EMR MKT ETF,900,$50.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$28.96,Cash,
Z00000001,BBBB,VOO,VANGUARD S&P 500 ETF,800,$40.922,+$0.160,"$45,900.35",+$150.25,+0.32%,"+$11,945.20",+31.10%,15.05%,"$38,362.05",$18.96,Cash,
""",
] {
var rejectedRows = [MHolding.Row]()
Expand All @@ -114,8 +114,18 @@ final class FidoPositionsTests: XCTestCase {
]

XCTAssertTrue(areEqual(expected2, actual2))
// XCTAssertEqual(expected2, actual2)
XCTAssertEqual(0, rejectedRows.count)

let actual3: [MHolding.Row] = try imp.decode(MHolding.self, dataStr, rejectedRows: &rejectedRows, outputSchema: .allocAccount)

let expected3: [MHolding.Row] = [
["accountID": "Z00000000", "title": "AAAA"],
["accountID": "Z00000001", "title": "BBBB"],
]

XCTAssertTrue(areEqual(expected3, actual3))
XCTAssertEqual(0, rejectedRows.count)

}
}
}

0 comments on commit 97c8a23

Please sign in to comment.