Loading ahci/ahci_dma.v +1 −1 Original line number Original line Diff line number Diff line Loading @@ -497,7 +497,7 @@ module ahci_dma ( .dout (sys_out), // output[31:0] .dout (sys_out), // output[31:0] .dout_vld (sys_dav), // output .dout_vld (sys_dav), // output .dout_re (sys_re), // input .dout_re (sys_re), // input .last_data (last_h2d_data) // output .last_DW (last_h2d_data) // output ); ); ahci_dma_wr_fifo #( // device to memory ahci_dma_wr_fifo #( // device to memory Loading ahci/ahci_dma_rd_fifo.v +88 −37 Original line number Original line Diff line number Diff line Loading @@ -60,7 +60,7 @@ module ahci_dma_rd_fifo#( output [31:0] dout, output [31:0] dout, output dout_vld, output dout_vld, input dout_re, input dout_re, output last_data // pulse @mclk (input done for the last prd - slow send out FIS, no data for 2 clocks - that was the last output last_DW // dout contains last DW ); ); localparam ADDRESS_NUM = (1<<ADDRESS_BITS); // 8 for ADDRESS_BITS==3 localparam ADDRESS_NUM = (1<<ADDRESS_BITS); // 8 for ADDRESS_BITS==3 reg [ADDRESS_BITS : 0] waddr; // 1 extra bit reg [ADDRESS_BITS : 0] waddr; // 1 extra bit Loading @@ -68,67 +68,101 @@ module ahci_dma_rd_fifo#( reg [63:16] din_prev; // only 48 bits are needed reg [63:16] din_prev; // only 48 bits are needed reg [WCNT_BITS-3:0] qwcntr; reg [WCNT_BITS-3:0] qwcntr; // reg some_offs; // reg some_offs; reg extra_in; /// reg extra_in; reg busy; reg busy; // reg din_last_w = din_re && (qwcntr==0); // reg din_last_w = din_re && (qwcntr==0); wire [2:0] end_offs = wcnt[1:0] + woffs; wire [2:0] end_offs = wcnt[1:0] + woffs; reg [63:0] fifo_ram [0: ADDRESS_NUM - 1]; reg [63:0] fifo_ram [0: ADDRESS_NUM - 1]; reg [3:0] vld_ram [0: ADDRESS_NUM - 1]; reg [3:0] vld_ram [0: ADDRESS_NUM - 1]; reg [1:0] flush_ram [0: ADDRESS_NUM - 1]; // reg [1:0] flush_ram [0: ADDRESS_NUM - 1]; reg [(1<<ADDRESS_BITS)-1:0] fifo_full; // set in write clock domain reg [(1<<ADDRESS_BITS)-1:0] fifo_full; // set in write clock domain reg [(1<<ADDRESS_BITS)-1:0] fifo_nempty;// set in read clock domain reg [(1<<ADDRESS_BITS)-1:0] fifo_nempty;// set in read clock domain wire fifo_wr; wire fifo_wr; wire fifo_rd; wire fifo_rd; reg hrst_mclk; reg [1:0] fifo_rd_r; wire [(1<<ADDRESS_BITS)-1:0] fifo_full2 = {fifo_full[0],fifo_full[ADDRESS_NUM-1:1]}; // reg hrst_mclk; reg mrst_hclk; /// wire [(1<<ADDRESS_BITS)-1:0] fifo_full2 = {fifo_full[0],fifo_full[ADDRESS_NUM-1:1]}; wire [(1<<ADDRESS_BITS)-1:0] fifo_full2 = {~fifo_full[0],fifo_full[ADDRESS_NUM-1:1]}; // wire [(1<<ADDRESS_BITS)-1:0] fifo_nempty_half = {fifo_nempty[(ADDRESS_NUM>>1)-1:0],fifo_full[ADDRESS_NUM-1: ADDRESS_NUM>>1]}; // wire [(1<<ADDRESS_BITS)-1:0] fifo_nempty_half = {fifo_nempty[(ADDRESS_NUM>>1)-1:0],fifo_full[ADDRESS_NUM-1: ADDRESS_NUM>>1]}; reg fifo_dav; // @mclk reg fifo_dav; // @mclk wire fifo_dav2_w; reg fifo_dav2; // @mclk reg fifo_dav2; // @mclk // wire fifo_dav_w; reg fifo_half_hclk; // Half Fifo is empty, OK to write reg fifo_half_hclk; // Half Fifo is empty, OK to write reg [1:0] woffs_r; reg [1:0] woffs_r; wire [63:0] fifo_di= woffs_r[1]?(woffs_r[0] ? {din[47:0],din_prev[63:48]} : {din[31:0],din_prev[63:32]}): wire [63:0] fifo_di= woffs_r[1]?(woffs_r[0] ? {din[47:0],din_prev[63:48]} : {din[31:0],din_prev[63:32]}): (woffs_r[0] ? {din[15:0],din_prev[63:16]} : din[63:0]); (woffs_r[0] ? {din[15:0],din_prev[63:16]} : din[63:0]); /// (woffs_r[0] ? {din[15:0],din_prev[63:16]} : din_prev[63:0]); wire [3:0] fifo_di_vld; wire [3:0] fifo_di_vld; wire [1:0] fifo_di_flush; // Assign // wire [1:0] fifo_di_flush; // Assign wire [63:0] fifo_do = fifo_ram [raddr[ADDRESS_BITS:1]]; wire [63:0] fifo_do = fifo_ram [raddr[ADDRESS_BITS:1]]; // wire [3:0] fifo_do_vld = fifo_dav_w? vld_ram [raddr[ADDRESS_BITS:1]] : 4'b0; wire [3:0] fifo_do_vld = vld_ram [raddr[ADDRESS_BITS:1]]; wire [3:0] fifo_do_vld = vld_ram [raddr[ADDRESS_BITS:1]]; wire [1:0] fifo_do_flush = flush_ram[raddr[ADDRESS_BITS:1]]; // wire [1:0] fifo_do_flush = fifo_dav_w? flush_ram[raddr[ADDRESS_BITS:1]] : 2'b0; reg din_av_safe_r; reg din_av_safe_r; reg en_fifo_wr; reg en_fifo_wr; reg [3:0] last_mask; reg [3:0] last_mask; reg flush_r; // reg flush_r; wire done_flush_mclk; wire done_flush_mclk; reg flushing_hclk; // flushing data, ends when confirmed from mclk domain reg flushing_mclk; // just registered flushing_hclk @mclk wire last_fifo_wr; assign din_re = busy && fifo_half_hclk && din_av_safe_r; assign din_re = busy && fifo_half_hclk && din_av_safe_r; assign fifo_wr = en_fifo_wr && fifo_half_hclk && (din_av_safe_r || !busy); assign fifo_wr = en_fifo_wr && fifo_half_hclk && (din_av_safe_r || !busy); assign fifo_di_vld = (busy && (!extra_in || (qwcntr != 0)))? 4'hf : last_mask ; /// assign fifo_di_vld = (busy && (!extra_in || (qwcntr != 0)))? 4'hf : last_mask ; assign fifo_di_flush = ((busy && (!extra_in || (qwcntr != 0))) || !flush_r)? 2'h0 : {|last_mask[3:2], ~(|last_mask[3:2])} ; /// assign fifo_di_flush = ((busy && (!extra_in || (qwcntr != 0))) || !flush_r)? 2'h0 : {|last_mask[3:2], ~(|last_mask[3:2])} ; /// assign fifo_di_vld = (busy && (qwcntr != 0))? 4'hf : last_mask ; assign fifo_di_vld = last_fifo_wr? last_mask : 4'hf; // assign fifo_di_flush = ((busy && (qwcntr != 0)) || !flush_r)? 2'h0 : {|last_mask[3:2], ~(|last_mask[3:2])} ; // assign fifo_dav_w = fifo_dav && (fifo_dav2 || !(|fifo_rd_r)); wire [2:0] debug_waddr = waddr[2:0]; wire [2:0] debug_raddr = raddr[3:1]; assign fifo_dav2_w = fifo_full2[raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; assign last_fifo_wr = !busy || ((qwcntr == 0) && ((woffs == 0) || end_offs[2])); // ((qwcntr != 0) || ((woffs != 0) && last_prd)); always @ (posedge hclk) begin always @ (posedge hclk) begin if (hrst) busy <= 0; if (hrst) mrst_hclk <= 0; else mrst_hclk <= mrst; if (mrst_hclk) busy <= 0; else if (start) busy <= 1; else if (start) busy <= 1; else if (din_re && (qwcntr == 0)) busy <= 0; else if (din_re && (qwcntr == 0)) busy <= 0; done <= busy && din_re && (qwcntr == 0); done <= busy && din_re && (qwcntr == 0); if (hrst) en_fifo_wr <= 0; if (mrst_hclk) en_fifo_wr <= 0; else if (start) en_fifo_wr <= (wcnt[1:0] == 0); else if (start) en_fifo_wr <= (woffs == 0); else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || extra_in); /// else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || ((woffs != 0) && last_prd)); else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || ((woffs != 0) && !end_offs[2])); //last_fifo_wr if (start) qwcntr <= wcnt[WCNT_BITS-1:2]; /// if (start) qwcntr <= wcnt[WCNT_BITS-1:2]; if (start) qwcntr <= wcnt[WCNT_BITS-1:2] + end_offs[2]; else if (din_re) qwcntr <= qwcntr - 1; else if (din_re) qwcntr <= qwcntr - 1; if (start) extra_in <= end_offs[2]; /// if (start) extra_in <= end_offs[2]; if (start) woffs_r <= woffs; if (start) woffs_r <= woffs; if (hrst) fifo_full <= 0; if (mrst_hclk) fifo_full <= 0; else if (fifo_wr) fifo_full <= {fifo_full[ADDRESS_NUM-2:0],waddr[ADDRESS_BITS]}; /// else if (fifo_wr) fifo_full <= {fifo_full[ADDRESS_NUM-2:0], waddr[ADDRESS_BITS]}; else if (fifo_wr) fifo_full <= {fifo_full[ADDRESS_NUM-2:0],~waddr[ADDRESS_BITS]}; if (hrst) waddr <= 0; if (mrst_hclk) waddr <= 0; else if (fifo_wr) waddr <= waddr+1; else if (fifo_wr) waddr <= waddr+1; fifo_half_hclk <= fifo_nempty [waddr[ADDRESS_BITS-1:0]] ^ waddr[ADDRESS_BITS]; fifo_half_hclk <= fifo_nempty [waddr[ADDRESS_BITS-1:0]] ^ waddr[ADDRESS_BITS]; Loading @@ -137,43 +171,59 @@ module ahci_dma_rd_fifo#( if (fifo_wr) fifo_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di; if (fifo_wr) fifo_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di; if (fifo_wr) vld_ram [waddr[ADDRESS_BITS-1:0]] <= fifo_di_vld; if (fifo_wr) vld_ram [waddr[ADDRESS_BITS-1:0]] <= fifo_di_vld; if (fifo_wr) flush_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di_flush; // if (fifo_wr) flush_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di_flush; if (hrst) din_av_safe_r <= 0; if (mrst_hclk) din_av_safe_r <= 0; else din_av_safe_r <= din_av && (din_av_many || !din_re); else din_av_safe_r <= din_av && (din_av_many || !din_re); if (start) last_mask <= {&wcnt, wcnt[1], |wcnt, 1'b1}; if (start) last_mask <= {&wcnt, wcnt[1], |wcnt, 1'b1}; if (start) flush_r <= last_prd; // if (start) flush_r <= last_prd; if (mrst_hclk || done_flush) flushing_hclk <= 0; // else if (busy && din_re && (qwcntr == 0) && last_prd) flushing_hclk <= 1; else if (fifo_wr && last_prd && (((qwcntr == 0) && ((woffs == 0) || !last_prd)) || !busy)) flushing_hclk <= 1; // else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || (woffs != 0)); end end always @ (posedge mclk) begin always @ (posedge mclk) begin hrst_mclk <= hrst; fifo_rd_r <= {fifo_rd_r[0],fifo_rd}; /// hrst_mclk <= hrst; if (hrst_mclk) raddr <= 0; /// if (hrst_mclk) raddr <= 0; if (mrst) raddr <= 0; else if (fifo_rd) raddr <= raddr + 1; else if (fifo_rd) raddr <= raddr + 1; if (hrst_mclk) fifo_nempty <= {{(ADDRESS_NUM>>1){1'b0}},{(ADDRESS_NUM>>1){1'b1}}};// 8'b00001111 /// if (hrst_mclk) fifo_nempty <= {{(ADDRESS_NUM>>1){1'b0}},{(ADDRESS_NUM>>1){1'b1}}};// 8'b00001111 else if (fifo_rd && raddr[0]) fifo_nempty <= {fifo_nempty[ADDRESS_NUM-2:0],raddr[ADDRESS_BITS+1] ^ raddr[ADDRESS_BITS]}; if (mrst) fifo_nempty <= {{(ADDRESS_NUM>>1){1'b0}},{(ADDRESS_NUM>>1){1'b1}}};// 8'b00001111 /// else if (fifo_rd && raddr[0]) fifo_nempty <= {fifo_nempty[ADDRESS_NUM-2:0],raddr[ADDRESS_BITS+1] ^ raddr[ADDRESS_BITS]}; else if (fifo_rd && raddr[0]) fifo_nempty <= {fifo_nempty[ADDRESS_NUM-2:0], ~raddr[ADDRESS_BITS+1] ^ raddr[ADDRESS_BITS]}; fifo_dav <= fifo_full [raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; fifo_dav <= fifo_full [raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; fifo_dav2 <= fifo_full2[raddr[ADDRESS_BITS:1]]; fifo_dav2 <= fifo_dav2_w; // fifo_full2[raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; if (mrst) flushing_mclk <= 0; else flushing_mclk <= flushing_hclk; end end ahci_dma_rd_stuff ahci_dma_rd_stuff_i ( ahci_dma_rd_stuff ahci_dma_rd_stuff_i ( .rst (mrst), // input .rst (mrst), // input .clk (mclk), // input .clk (mclk), // input .din_av (fifo_dav), // input .din_av (fifo_dav), // input .din_avm_w(fifo_dav2_w), // input .din_avm (fifo_dav2), // input .din_avm (fifo_dav2), // input .flush (raddr[0]?fifo_do_flush[1]:fifo_do_flush[0]), // input // .flush (raddr[0]?fifo_do_flush[1]:fifo_do_flush[0]), // input .flushing (flushing_mclk), // input .din (raddr[0]?fifo_do[63:32]: fifo_do[31:0]), // input[31:0] .din (raddr[0]?fifo_do[63:32]: fifo_do[31:0]), // input[31:0] .dm (raddr[0]?fifo_do_vld[3:2]:fifo_do_vld[1:0]), // input[1:0] .dm (raddr[0]?fifo_do_vld[3:2]:fifo_do_vld[1:0]), // input[1:0] .din_re (fifo_rd), // output .din_re (fifo_rd), // output .flushed (done_flush_mclk), // output reg: flush (end of last PRD is finished - data left module) .flushed (done_flush_mclk), // output reg: flush (end of last PRD is finished - data left module) .dout (dout), // output[31:0] reg .dout (dout), // output[31:0] reg .dout_vld (dout_vld), // output .dout_vld (dout_vld), // output .dout_re (dout_re) // input .dout_re (dout_re), // input .last_DW (last_DW) ); ); pulse_cross_clock #( pulse_cross_clock #( Loading @@ -182,20 +232,21 @@ module ahci_dma_rd_fifo#( .rst (mrst), // input .rst (mrst), // input .src_clk (mclk), // input .src_clk (mclk), // input .dst_clk (hclk), // input .dst_clk (hclk), // input .in_pulse (flush_r && din_re && (qwcntr == 0)), // input // .in_pulse (flush_r && din_re && (qwcntr == 0)), // input .in_pulse (done_flush_mclk), // input .out_pulse (done_flush), // output .out_pulse (done_flush), // output .busy() // output .busy() // output ); ); /* pulse_cross_clock #( pulse_cross_clock #( .EXTRA_DLY(0) .EXTRA_DLY(0) ) last_data_i ( ) last_data_i ( .rst (mrst), // input .rst (mrst_hclk), // input .src_clk (mclk), // input .src_clk (hclk), // input .dst_clk (hclk), // input .dst_clk (mclk), // input .in_pulse (done_flush_mclk), // input .in_pulse (busy && din_re && (qwcntr == 0) && last_prd),// input .out_pulse (last_data), // output .out_pulse (last_data), // output .busy() // output .busy() // output ); ); */ endmodule endmodule ahci/ahci_dma_rd_stuff.v +101 −36 Original line number Original line Diff line number Diff line Loading @@ -37,59 +37,124 @@ module ahci_dma_rd_stuff( input rst, // sync reset input rst, // sync reset input clk, // single clock input clk, // single clock input din_av, // input data available input din_av, // input data available input din_avm, // >1 word of data available input din_avm_w,// >1 word of data available (early) input flush, // output partial dword if available (should be ? cycles after last _re/ with data?) input din_avm, // >1 word of data available (registered din_avm_w) input flushing, // output partial dword if available (should be ? cycles after last _re/ with data?) input [31:0] din, // 32-bit input dfata input [31:0] din, // 32-bit input dfata input [1:0] dm, // data mask showing which (if any) words in input dword are valid input [1:0] dm, // data mask showing which (if any) words in input dword are valid output din_re, // read input data output din_re, // read input data output reg flushed, // flush (end of last PRD is finished - data left module) output flushed, // flush (end of last PRD is finished - data left module) output reg [31:0] dout, // output 32-bit data output reg [31:0] dout, // output 32-bit data output dout_vld, // output data valid output dout_vld, // output data valid input dout_re // consumer reads output data (should be AND-ed with dout_vld) input dout_re, // consumer reads output data (should be AND-ed with dout_vld) output last_DW ); ); reg [15:0] hr; // holds 16-bit data from previous din_re if not consumed reg [15:0] hr; // holds 16-bit data from previous din_re if not consumed reg hr_full; reg hr_full; reg dout_vld_r; reg [1:0] dout_vld_r; reg flushing; reg flushing_d; reg din_av_safe_r; reg din_av_safe_r; reg din_re_r; wire [1:0] dav_in = {2{din_av_safe_r}} & dm; wire [1:0] dav_in = {2{din_av_safe_r}} & dm; wire two_words_avail = &dav_in || (|dav_in && hr_full); wire [1:0] drd_in = {2{din_re}} & dm; assign din_re = (din_av_safe_r && !(|dm)) || ((!dout_vld_r || dout_re) && (two_words_avail)) ; // flush assign dout_vld = dout_vld_r; wire [15:0] debug_din_low = din[15: 0]; wire [15:0] debug_din_high = din[31:16]; wire [15:0] debug_dout_low = dout[15: 0]; wire [15:0] debug_dout_high = dout[31:16]; // wire empty_in = din_av_safe_r && !(|dm); // wire two_words_avail = &dav_in || (|dav_in && hr_full); wire more_words_avail = |dav_in || hr_full; wire [1:0] next_or_empty = {2{dout_re}} | ~dout_vld_r; /// assign din_re = (din_av_safe_r && !(|dm)) || ((!dout_vld_r || dout_re) && (two_words_avail)) ; // flush // --------------- wire room_for2 = dout_re || (!(&dout_vld_r) && !hr_full) || !(|dout_vld_r); wire room_for1 = dout_re || !hr_full || !(&dout_vld_r); reg slow_down; // first time fifo almost empty reg slow_dav; // enable dout_vld waiting after each read out not to miss last DWORD reg last_DW_r; reg last_dw_sent; wire no_new_data_w; reg [1:0] no_new_data_r; assign din_re = din_av_safe_r && (!(|dm) || room_for2 || (room_for1 && !(&dm))); /// assign dout_vld = (&dout_vld_r) || ((|dout_vld_r) && flushing); assign dout_vld = (!slow_down && (&dout_vld_r)) || slow_dav; assign last_DW = last_DW_r; assign flushed = last_DW_r && dout_re; assign no_new_data_w = !din_av && !hr_full; // assign flushed = always @ (posedge clk) begin always @ (posedge clk) begin din_re_r <= din_re; if (rst) din_av_safe_r <= 0; if (rst) din_av_safe_r <= 0; else din_av_safe_r <= din_av && (din_avm || !din_re); else din_av_safe_r <= din_av && (din_avm || (!din_re && !din_re_r)); // set low word of the OR if (rst) dout_vld_r[0] <= 0; else if (next_or_empty[0]) dout_vld_r[0] <= hr_full || (din_re && (|dm)); if ((!dout_vld_r || dout_re) && (two_words_avail || flushing)) begin if (next_or_empty[0]) begin if (hr_full) dout[15: 0] <= hr; if (hr_full) dout[15: 0] <= hr; else dout[15: 0] <= din[15: 0]; else if (din_re) begin if (dm[0]) dout[15: 0] <= din[15: 0]; else if (dm[1]) dout[15: 0] <= din[31:16]; end end // set high word of the OR if (rst) dout_vld_r[1] <= 0; else if (next_or_empty[1]) dout_vld_r[1] <= next_or_empty[0]? (din_re && ((hr_full &&(|dm)) || (&dm))) : (hr_full || (din_re && (|dm))); if (hr_full && dav_in[0]) dout[31:16] <= din[15: 0]; if (next_or_empty[1]) begin else dout[31:16] <= din[31:16]; if (next_or_empty[0]) begin if (din_re) begin if (hr_full && dm[0]) dout[31:16] <= din[15: 0]; else if (dm[1] && (!hr_full || dm[0])) dout[31:16] <= din[31:16]; end end else begin if (hr_full) dout[31:16] <= hr; else if (din_re) begin if (dm[0]) dout[31:16] <= din[15: 0]; else if (dm[1]) dout[31:16] <= din[31:16]; end end end end // todo add reset/flush // set holding register if (rst) hr_full <= 0; if (rst) hr_full <= 0; else if (!dout_vld_r || dout_re) else if (((&next_or_empty) && !(&drd_in)) || // 2 but not 3 sources available ((|next_or_empty) && !(|drd_in))) hr_full <= 0; if (flushing || ((two_words_avail) && ! (&dav_in && hr_full))) hr_full <= 0; else if (((&drd_in) && !(&next_or_empty)) || else if (dav_in[0] ^ dav_in[1]) hr_full <= 1; ((|drd_in) && !(|next_or_empty))) hr_full <= 1; if (drd_in[1]) hr <= din[31:16]; else if (drd_in[0]) hr <= din[15: 0]; if (rst || !flushing) slow_down <= 0; else if (!din_avm_w) slow_down <= 1; if ((!dout_vld_r || dout_re) && (&dav_in && hr_full)) hr <= din[31:16]; if (rst || !flushing || last_dw_sent) slow_dav <= 0; else if ((dav_in[0] ^ dav_in[1]) && !hr_full) hr <= dav_in[0]? din[15:0] : din[31:16]; else slow_dav <= !dout_re && !last_dw_sent && ((!next_or_empty[1] && more_words_avail) || last_DW_r); if (rst) dout_vld_r <= 0; else if ((!dout_vld_r || dout_re) && (two_words_avail || (flushing && hr_full))) dout_vld_r <= 1; else if (dout_re) dout_vld_r <= 0; if (rst) flushing <= 0; if (rst || !flushing) last_dw_sent <= 0; else if (flush) flushing <= 1; else if (last_DW_r && dout_re) last_dw_sent <= 1; else if ((!dout_vld_r || dout_re) && !(&dav_in && hr_full)) flushing <= 0; flushing_d <= flushing; no_new_data_r <= {no_new_data_r[0], no_new_data_w}; if (rst || !flushing) last_DW_r <= 0; else if (slow_down && no_new_data_w && (&no_new_data_r)) last_DW_r <= 1; else if (dout_re) last_DW_r <= 0; flushed <= flushing_d && !flushing; // 1 cycle delay end end endmodule endmodule Loading Loading
ahci/ahci_dma.v +1 −1 Original line number Original line Diff line number Diff line Loading @@ -497,7 +497,7 @@ module ahci_dma ( .dout (sys_out), // output[31:0] .dout (sys_out), // output[31:0] .dout_vld (sys_dav), // output .dout_vld (sys_dav), // output .dout_re (sys_re), // input .dout_re (sys_re), // input .last_data (last_h2d_data) // output .last_DW (last_h2d_data) // output ); ); ahci_dma_wr_fifo #( // device to memory ahci_dma_wr_fifo #( // device to memory Loading
ahci/ahci_dma_rd_fifo.v +88 −37 Original line number Original line Diff line number Diff line Loading @@ -60,7 +60,7 @@ module ahci_dma_rd_fifo#( output [31:0] dout, output [31:0] dout, output dout_vld, output dout_vld, input dout_re, input dout_re, output last_data // pulse @mclk (input done for the last prd - slow send out FIS, no data for 2 clocks - that was the last output last_DW // dout contains last DW ); ); localparam ADDRESS_NUM = (1<<ADDRESS_BITS); // 8 for ADDRESS_BITS==3 localparam ADDRESS_NUM = (1<<ADDRESS_BITS); // 8 for ADDRESS_BITS==3 reg [ADDRESS_BITS : 0] waddr; // 1 extra bit reg [ADDRESS_BITS : 0] waddr; // 1 extra bit Loading @@ -68,67 +68,101 @@ module ahci_dma_rd_fifo#( reg [63:16] din_prev; // only 48 bits are needed reg [63:16] din_prev; // only 48 bits are needed reg [WCNT_BITS-3:0] qwcntr; reg [WCNT_BITS-3:0] qwcntr; // reg some_offs; // reg some_offs; reg extra_in; /// reg extra_in; reg busy; reg busy; // reg din_last_w = din_re && (qwcntr==0); // reg din_last_w = din_re && (qwcntr==0); wire [2:0] end_offs = wcnt[1:0] + woffs; wire [2:0] end_offs = wcnt[1:0] + woffs; reg [63:0] fifo_ram [0: ADDRESS_NUM - 1]; reg [63:0] fifo_ram [0: ADDRESS_NUM - 1]; reg [3:0] vld_ram [0: ADDRESS_NUM - 1]; reg [3:0] vld_ram [0: ADDRESS_NUM - 1]; reg [1:0] flush_ram [0: ADDRESS_NUM - 1]; // reg [1:0] flush_ram [0: ADDRESS_NUM - 1]; reg [(1<<ADDRESS_BITS)-1:0] fifo_full; // set in write clock domain reg [(1<<ADDRESS_BITS)-1:0] fifo_full; // set in write clock domain reg [(1<<ADDRESS_BITS)-1:0] fifo_nempty;// set in read clock domain reg [(1<<ADDRESS_BITS)-1:0] fifo_nempty;// set in read clock domain wire fifo_wr; wire fifo_wr; wire fifo_rd; wire fifo_rd; reg hrst_mclk; reg [1:0] fifo_rd_r; wire [(1<<ADDRESS_BITS)-1:0] fifo_full2 = {fifo_full[0],fifo_full[ADDRESS_NUM-1:1]}; // reg hrst_mclk; reg mrst_hclk; /// wire [(1<<ADDRESS_BITS)-1:0] fifo_full2 = {fifo_full[0],fifo_full[ADDRESS_NUM-1:1]}; wire [(1<<ADDRESS_BITS)-1:0] fifo_full2 = {~fifo_full[0],fifo_full[ADDRESS_NUM-1:1]}; // wire [(1<<ADDRESS_BITS)-1:0] fifo_nempty_half = {fifo_nempty[(ADDRESS_NUM>>1)-1:0],fifo_full[ADDRESS_NUM-1: ADDRESS_NUM>>1]}; // wire [(1<<ADDRESS_BITS)-1:0] fifo_nempty_half = {fifo_nempty[(ADDRESS_NUM>>1)-1:0],fifo_full[ADDRESS_NUM-1: ADDRESS_NUM>>1]}; reg fifo_dav; // @mclk reg fifo_dav; // @mclk wire fifo_dav2_w; reg fifo_dav2; // @mclk reg fifo_dav2; // @mclk // wire fifo_dav_w; reg fifo_half_hclk; // Half Fifo is empty, OK to write reg fifo_half_hclk; // Half Fifo is empty, OK to write reg [1:0] woffs_r; reg [1:0] woffs_r; wire [63:0] fifo_di= woffs_r[1]?(woffs_r[0] ? {din[47:0],din_prev[63:48]} : {din[31:0],din_prev[63:32]}): wire [63:0] fifo_di= woffs_r[1]?(woffs_r[0] ? {din[47:0],din_prev[63:48]} : {din[31:0],din_prev[63:32]}): (woffs_r[0] ? {din[15:0],din_prev[63:16]} : din[63:0]); (woffs_r[0] ? {din[15:0],din_prev[63:16]} : din[63:0]); /// (woffs_r[0] ? {din[15:0],din_prev[63:16]} : din_prev[63:0]); wire [3:0] fifo_di_vld; wire [3:0] fifo_di_vld; wire [1:0] fifo_di_flush; // Assign // wire [1:0] fifo_di_flush; // Assign wire [63:0] fifo_do = fifo_ram [raddr[ADDRESS_BITS:1]]; wire [63:0] fifo_do = fifo_ram [raddr[ADDRESS_BITS:1]]; // wire [3:0] fifo_do_vld = fifo_dav_w? vld_ram [raddr[ADDRESS_BITS:1]] : 4'b0; wire [3:0] fifo_do_vld = vld_ram [raddr[ADDRESS_BITS:1]]; wire [3:0] fifo_do_vld = vld_ram [raddr[ADDRESS_BITS:1]]; wire [1:0] fifo_do_flush = flush_ram[raddr[ADDRESS_BITS:1]]; // wire [1:0] fifo_do_flush = fifo_dav_w? flush_ram[raddr[ADDRESS_BITS:1]] : 2'b0; reg din_av_safe_r; reg din_av_safe_r; reg en_fifo_wr; reg en_fifo_wr; reg [3:0] last_mask; reg [3:0] last_mask; reg flush_r; // reg flush_r; wire done_flush_mclk; wire done_flush_mclk; reg flushing_hclk; // flushing data, ends when confirmed from mclk domain reg flushing_mclk; // just registered flushing_hclk @mclk wire last_fifo_wr; assign din_re = busy && fifo_half_hclk && din_av_safe_r; assign din_re = busy && fifo_half_hclk && din_av_safe_r; assign fifo_wr = en_fifo_wr && fifo_half_hclk && (din_av_safe_r || !busy); assign fifo_wr = en_fifo_wr && fifo_half_hclk && (din_av_safe_r || !busy); assign fifo_di_vld = (busy && (!extra_in || (qwcntr != 0)))? 4'hf : last_mask ; /// assign fifo_di_vld = (busy && (!extra_in || (qwcntr != 0)))? 4'hf : last_mask ; assign fifo_di_flush = ((busy && (!extra_in || (qwcntr != 0))) || !flush_r)? 2'h0 : {|last_mask[3:2], ~(|last_mask[3:2])} ; /// assign fifo_di_flush = ((busy && (!extra_in || (qwcntr != 0))) || !flush_r)? 2'h0 : {|last_mask[3:2], ~(|last_mask[3:2])} ; /// assign fifo_di_vld = (busy && (qwcntr != 0))? 4'hf : last_mask ; assign fifo_di_vld = last_fifo_wr? last_mask : 4'hf; // assign fifo_di_flush = ((busy && (qwcntr != 0)) || !flush_r)? 2'h0 : {|last_mask[3:2], ~(|last_mask[3:2])} ; // assign fifo_dav_w = fifo_dav && (fifo_dav2 || !(|fifo_rd_r)); wire [2:0] debug_waddr = waddr[2:0]; wire [2:0] debug_raddr = raddr[3:1]; assign fifo_dav2_w = fifo_full2[raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; assign last_fifo_wr = !busy || ((qwcntr == 0) && ((woffs == 0) || end_offs[2])); // ((qwcntr != 0) || ((woffs != 0) && last_prd)); always @ (posedge hclk) begin always @ (posedge hclk) begin if (hrst) busy <= 0; if (hrst) mrst_hclk <= 0; else mrst_hclk <= mrst; if (mrst_hclk) busy <= 0; else if (start) busy <= 1; else if (start) busy <= 1; else if (din_re && (qwcntr == 0)) busy <= 0; else if (din_re && (qwcntr == 0)) busy <= 0; done <= busy && din_re && (qwcntr == 0); done <= busy && din_re && (qwcntr == 0); if (hrst) en_fifo_wr <= 0; if (mrst_hclk) en_fifo_wr <= 0; else if (start) en_fifo_wr <= (wcnt[1:0] == 0); else if (start) en_fifo_wr <= (woffs == 0); else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || extra_in); /// else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || ((woffs != 0) && last_prd)); else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || ((woffs != 0) && !end_offs[2])); //last_fifo_wr if (start) qwcntr <= wcnt[WCNT_BITS-1:2]; /// if (start) qwcntr <= wcnt[WCNT_BITS-1:2]; if (start) qwcntr <= wcnt[WCNT_BITS-1:2] + end_offs[2]; else if (din_re) qwcntr <= qwcntr - 1; else if (din_re) qwcntr <= qwcntr - 1; if (start) extra_in <= end_offs[2]; /// if (start) extra_in <= end_offs[2]; if (start) woffs_r <= woffs; if (start) woffs_r <= woffs; if (hrst) fifo_full <= 0; if (mrst_hclk) fifo_full <= 0; else if (fifo_wr) fifo_full <= {fifo_full[ADDRESS_NUM-2:0],waddr[ADDRESS_BITS]}; /// else if (fifo_wr) fifo_full <= {fifo_full[ADDRESS_NUM-2:0], waddr[ADDRESS_BITS]}; else if (fifo_wr) fifo_full <= {fifo_full[ADDRESS_NUM-2:0],~waddr[ADDRESS_BITS]}; if (hrst) waddr <= 0; if (mrst_hclk) waddr <= 0; else if (fifo_wr) waddr <= waddr+1; else if (fifo_wr) waddr <= waddr+1; fifo_half_hclk <= fifo_nempty [waddr[ADDRESS_BITS-1:0]] ^ waddr[ADDRESS_BITS]; fifo_half_hclk <= fifo_nempty [waddr[ADDRESS_BITS-1:0]] ^ waddr[ADDRESS_BITS]; Loading @@ -137,43 +171,59 @@ module ahci_dma_rd_fifo#( if (fifo_wr) fifo_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di; if (fifo_wr) fifo_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di; if (fifo_wr) vld_ram [waddr[ADDRESS_BITS-1:0]] <= fifo_di_vld; if (fifo_wr) vld_ram [waddr[ADDRESS_BITS-1:0]] <= fifo_di_vld; if (fifo_wr) flush_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di_flush; // if (fifo_wr) flush_ram[waddr[ADDRESS_BITS-1:0]] <= fifo_di_flush; if (hrst) din_av_safe_r <= 0; if (mrst_hclk) din_av_safe_r <= 0; else din_av_safe_r <= din_av && (din_av_many || !din_re); else din_av_safe_r <= din_av && (din_av_many || !din_re); if (start) last_mask <= {&wcnt, wcnt[1], |wcnt, 1'b1}; if (start) last_mask <= {&wcnt, wcnt[1], |wcnt, 1'b1}; if (start) flush_r <= last_prd; // if (start) flush_r <= last_prd; if (mrst_hclk || done_flush) flushing_hclk <= 0; // else if (busy && din_re && (qwcntr == 0) && last_prd) flushing_hclk <= 1; else if (fifo_wr && last_prd && (((qwcntr == 0) && ((woffs == 0) || !last_prd)) || !busy)) flushing_hclk <= 1; // else if (din_re || fifo_wr) en_fifo_wr <= busy && ((qwcntr != 0) || (woffs != 0)); end end always @ (posedge mclk) begin always @ (posedge mclk) begin hrst_mclk <= hrst; fifo_rd_r <= {fifo_rd_r[0],fifo_rd}; /// hrst_mclk <= hrst; if (hrst_mclk) raddr <= 0; /// if (hrst_mclk) raddr <= 0; if (mrst) raddr <= 0; else if (fifo_rd) raddr <= raddr + 1; else if (fifo_rd) raddr <= raddr + 1; if (hrst_mclk) fifo_nempty <= {{(ADDRESS_NUM>>1){1'b0}},{(ADDRESS_NUM>>1){1'b1}}};// 8'b00001111 /// if (hrst_mclk) fifo_nempty <= {{(ADDRESS_NUM>>1){1'b0}},{(ADDRESS_NUM>>1){1'b1}}};// 8'b00001111 else if (fifo_rd && raddr[0]) fifo_nempty <= {fifo_nempty[ADDRESS_NUM-2:0],raddr[ADDRESS_BITS+1] ^ raddr[ADDRESS_BITS]}; if (mrst) fifo_nempty <= {{(ADDRESS_NUM>>1){1'b0}},{(ADDRESS_NUM>>1){1'b1}}};// 8'b00001111 /// else if (fifo_rd && raddr[0]) fifo_nempty <= {fifo_nempty[ADDRESS_NUM-2:0],raddr[ADDRESS_BITS+1] ^ raddr[ADDRESS_BITS]}; else if (fifo_rd && raddr[0]) fifo_nempty <= {fifo_nempty[ADDRESS_NUM-2:0], ~raddr[ADDRESS_BITS+1] ^ raddr[ADDRESS_BITS]}; fifo_dav <= fifo_full [raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; fifo_dav <= fifo_full [raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; fifo_dav2 <= fifo_full2[raddr[ADDRESS_BITS:1]]; fifo_dav2 <= fifo_dav2_w; // fifo_full2[raddr[ADDRESS_BITS:1]] ^ raddr[ADDRESS_BITS+1]; if (mrst) flushing_mclk <= 0; else flushing_mclk <= flushing_hclk; end end ahci_dma_rd_stuff ahci_dma_rd_stuff_i ( ahci_dma_rd_stuff ahci_dma_rd_stuff_i ( .rst (mrst), // input .rst (mrst), // input .clk (mclk), // input .clk (mclk), // input .din_av (fifo_dav), // input .din_av (fifo_dav), // input .din_avm_w(fifo_dav2_w), // input .din_avm (fifo_dav2), // input .din_avm (fifo_dav2), // input .flush (raddr[0]?fifo_do_flush[1]:fifo_do_flush[0]), // input // .flush (raddr[0]?fifo_do_flush[1]:fifo_do_flush[0]), // input .flushing (flushing_mclk), // input .din (raddr[0]?fifo_do[63:32]: fifo_do[31:0]), // input[31:0] .din (raddr[0]?fifo_do[63:32]: fifo_do[31:0]), // input[31:0] .dm (raddr[0]?fifo_do_vld[3:2]:fifo_do_vld[1:0]), // input[1:0] .dm (raddr[0]?fifo_do_vld[3:2]:fifo_do_vld[1:0]), // input[1:0] .din_re (fifo_rd), // output .din_re (fifo_rd), // output .flushed (done_flush_mclk), // output reg: flush (end of last PRD is finished - data left module) .flushed (done_flush_mclk), // output reg: flush (end of last PRD is finished - data left module) .dout (dout), // output[31:0] reg .dout (dout), // output[31:0] reg .dout_vld (dout_vld), // output .dout_vld (dout_vld), // output .dout_re (dout_re) // input .dout_re (dout_re), // input .last_DW (last_DW) ); ); pulse_cross_clock #( pulse_cross_clock #( Loading @@ -182,20 +232,21 @@ module ahci_dma_rd_fifo#( .rst (mrst), // input .rst (mrst), // input .src_clk (mclk), // input .src_clk (mclk), // input .dst_clk (hclk), // input .dst_clk (hclk), // input .in_pulse (flush_r && din_re && (qwcntr == 0)), // input // .in_pulse (flush_r && din_re && (qwcntr == 0)), // input .in_pulse (done_flush_mclk), // input .out_pulse (done_flush), // output .out_pulse (done_flush), // output .busy() // output .busy() // output ); ); /* pulse_cross_clock #( pulse_cross_clock #( .EXTRA_DLY(0) .EXTRA_DLY(0) ) last_data_i ( ) last_data_i ( .rst (mrst), // input .rst (mrst_hclk), // input .src_clk (mclk), // input .src_clk (hclk), // input .dst_clk (hclk), // input .dst_clk (mclk), // input .in_pulse (done_flush_mclk), // input .in_pulse (busy && din_re && (qwcntr == 0) && last_prd),// input .out_pulse (last_data), // output .out_pulse (last_data), // output .busy() // output .busy() // output ); ); */ endmodule endmodule
ahci/ahci_dma_rd_stuff.v +101 −36 Original line number Original line Diff line number Diff line Loading @@ -37,59 +37,124 @@ module ahci_dma_rd_stuff( input rst, // sync reset input rst, // sync reset input clk, // single clock input clk, // single clock input din_av, // input data available input din_av, // input data available input din_avm, // >1 word of data available input din_avm_w,// >1 word of data available (early) input flush, // output partial dword if available (should be ? cycles after last _re/ with data?) input din_avm, // >1 word of data available (registered din_avm_w) input flushing, // output partial dword if available (should be ? cycles after last _re/ with data?) input [31:0] din, // 32-bit input dfata input [31:0] din, // 32-bit input dfata input [1:0] dm, // data mask showing which (if any) words in input dword are valid input [1:0] dm, // data mask showing which (if any) words in input dword are valid output din_re, // read input data output din_re, // read input data output reg flushed, // flush (end of last PRD is finished - data left module) output flushed, // flush (end of last PRD is finished - data left module) output reg [31:0] dout, // output 32-bit data output reg [31:0] dout, // output 32-bit data output dout_vld, // output data valid output dout_vld, // output data valid input dout_re // consumer reads output data (should be AND-ed with dout_vld) input dout_re, // consumer reads output data (should be AND-ed with dout_vld) output last_DW ); ); reg [15:0] hr; // holds 16-bit data from previous din_re if not consumed reg [15:0] hr; // holds 16-bit data from previous din_re if not consumed reg hr_full; reg hr_full; reg dout_vld_r; reg [1:0] dout_vld_r; reg flushing; reg flushing_d; reg din_av_safe_r; reg din_av_safe_r; reg din_re_r; wire [1:0] dav_in = {2{din_av_safe_r}} & dm; wire [1:0] dav_in = {2{din_av_safe_r}} & dm; wire two_words_avail = &dav_in || (|dav_in && hr_full); wire [1:0] drd_in = {2{din_re}} & dm; assign din_re = (din_av_safe_r && !(|dm)) || ((!dout_vld_r || dout_re) && (two_words_avail)) ; // flush assign dout_vld = dout_vld_r; wire [15:0] debug_din_low = din[15: 0]; wire [15:0] debug_din_high = din[31:16]; wire [15:0] debug_dout_low = dout[15: 0]; wire [15:0] debug_dout_high = dout[31:16]; // wire empty_in = din_av_safe_r && !(|dm); // wire two_words_avail = &dav_in || (|dav_in && hr_full); wire more_words_avail = |dav_in || hr_full; wire [1:0] next_or_empty = {2{dout_re}} | ~dout_vld_r; /// assign din_re = (din_av_safe_r && !(|dm)) || ((!dout_vld_r || dout_re) && (two_words_avail)) ; // flush // --------------- wire room_for2 = dout_re || (!(&dout_vld_r) && !hr_full) || !(|dout_vld_r); wire room_for1 = dout_re || !hr_full || !(&dout_vld_r); reg slow_down; // first time fifo almost empty reg slow_dav; // enable dout_vld waiting after each read out not to miss last DWORD reg last_DW_r; reg last_dw_sent; wire no_new_data_w; reg [1:0] no_new_data_r; assign din_re = din_av_safe_r && (!(|dm) || room_for2 || (room_for1 && !(&dm))); /// assign dout_vld = (&dout_vld_r) || ((|dout_vld_r) && flushing); assign dout_vld = (!slow_down && (&dout_vld_r)) || slow_dav; assign last_DW = last_DW_r; assign flushed = last_DW_r && dout_re; assign no_new_data_w = !din_av && !hr_full; // assign flushed = always @ (posedge clk) begin always @ (posedge clk) begin din_re_r <= din_re; if (rst) din_av_safe_r <= 0; if (rst) din_av_safe_r <= 0; else din_av_safe_r <= din_av && (din_avm || !din_re); else din_av_safe_r <= din_av && (din_avm || (!din_re && !din_re_r)); // set low word of the OR if (rst) dout_vld_r[0] <= 0; else if (next_or_empty[0]) dout_vld_r[0] <= hr_full || (din_re && (|dm)); if ((!dout_vld_r || dout_re) && (two_words_avail || flushing)) begin if (next_or_empty[0]) begin if (hr_full) dout[15: 0] <= hr; if (hr_full) dout[15: 0] <= hr; else dout[15: 0] <= din[15: 0]; else if (din_re) begin if (dm[0]) dout[15: 0] <= din[15: 0]; else if (dm[1]) dout[15: 0] <= din[31:16]; end end // set high word of the OR if (rst) dout_vld_r[1] <= 0; else if (next_or_empty[1]) dout_vld_r[1] <= next_or_empty[0]? (din_re && ((hr_full &&(|dm)) || (&dm))) : (hr_full || (din_re && (|dm))); if (hr_full && dav_in[0]) dout[31:16] <= din[15: 0]; if (next_or_empty[1]) begin else dout[31:16] <= din[31:16]; if (next_or_empty[0]) begin if (din_re) begin if (hr_full && dm[0]) dout[31:16] <= din[15: 0]; else if (dm[1] && (!hr_full || dm[0])) dout[31:16] <= din[31:16]; end end else begin if (hr_full) dout[31:16] <= hr; else if (din_re) begin if (dm[0]) dout[31:16] <= din[15: 0]; else if (dm[1]) dout[31:16] <= din[31:16]; end end end end // todo add reset/flush // set holding register if (rst) hr_full <= 0; if (rst) hr_full <= 0; else if (!dout_vld_r || dout_re) else if (((&next_or_empty) && !(&drd_in)) || // 2 but not 3 sources available ((|next_or_empty) && !(|drd_in))) hr_full <= 0; if (flushing || ((two_words_avail) && ! (&dav_in && hr_full))) hr_full <= 0; else if (((&drd_in) && !(&next_or_empty)) || else if (dav_in[0] ^ dav_in[1]) hr_full <= 1; ((|drd_in) && !(|next_or_empty))) hr_full <= 1; if (drd_in[1]) hr <= din[31:16]; else if (drd_in[0]) hr <= din[15: 0]; if (rst || !flushing) slow_down <= 0; else if (!din_avm_w) slow_down <= 1; if ((!dout_vld_r || dout_re) && (&dav_in && hr_full)) hr <= din[31:16]; if (rst || !flushing || last_dw_sent) slow_dav <= 0; else if ((dav_in[0] ^ dav_in[1]) && !hr_full) hr <= dav_in[0]? din[15:0] : din[31:16]; else slow_dav <= !dout_re && !last_dw_sent && ((!next_or_empty[1] && more_words_avail) || last_DW_r); if (rst) dout_vld_r <= 0; else if ((!dout_vld_r || dout_re) && (two_words_avail || (flushing && hr_full))) dout_vld_r <= 1; else if (dout_re) dout_vld_r <= 0; if (rst) flushing <= 0; if (rst || !flushing) last_dw_sent <= 0; else if (flush) flushing <= 1; else if (last_DW_r && dout_re) last_dw_sent <= 1; else if ((!dout_vld_r || dout_re) && !(&dav_in && hr_full)) flushing <= 0; flushing_d <= flushing; no_new_data_r <= {no_new_data_r[0], no_new_data_w}; if (rst || !flushing) last_DW_r <= 0; else if (slow_down && no_new_data_w && (&no_new_data_r)) last_DW_r <= 1; else if (dout_re) last_DW_r <= 0; flushed <= flushing_d && !flushing; // 1 cycle delay end end endmodule endmodule Loading