From 3bd76e879bad8dc99cf8e785df6fc6ff84a502db Mon Sep 17 00:00:00 2001
From: rjianu <rjianu@abcam.com>
Date: Wed, 12 Jul 2023 16:26:54 +0300
Subject: [PATCH] feat: Improve memory allocation

---
 csv.go | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/csv.go b/csv.go
index eef1e0c..71014fe 100644
--- a/csv.go
+++ b/csv.go
@@ -23,16 +23,22 @@ func avg(data []float64) float64 {
 
 func csv2float(r io.Reader, column int) ([]float64, error) {
 	cr := csv.NewReader(r)
+	// set this to reuse the same slice for each read
+	// operation to reduce the memory allocation
+	cr.ReuseRecord = true
 	if column < 1 {
 		return nil, fmt.Errorf("%w: please provide a valid column number", ErrInvalidColumn)
 	}
 	column--
-	allData, err := cr.ReadAll()
-	if err != nil {
-		return nil, fmt.Errorf("cannot read data from file: %w", err)
-	}
 	var data []float64
-	for i, row := range allData {
+	for i := 0; ; i++ {
+		row, err := cr.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("cannot read data from file: %w", err)
+		}
 		if i == 0 {
 			continue
 		}