Takes too long to get address

I've simplified the code a little bit to hopefully make some things happen faster/cleaner but it still misses that first value.

library ieee;
use ieee.std_logic_1164.all;
use ieee.std_logic_arith.all;
use ieee.std_logic_unsigned.all; 
entity framestore is
	port
	(
		clk,reset : in std_logic;
		input: in std_logic_vector(3 downto 0); -- the input ethernet frame
		valid: in std_logic; --valid signal that comes with input
		readEN: in std_logic; --high when we want to read from memory
		rwaddressstart: in std_logic_vector(14 downto 0); --address where we want to START reading or writing
		output : out std_logic_vector(3 downto 0); --output, has value if we're reading
		addressout: out std_logic_vector(14 downto 0); --test signal to tell us the address
		holdout: out std_logic_vector(3 downto 0) --test signal to tell us what's being read out of memory
	);
end framestore;
architecture structure of framestore is
	signal hold: std_logic_vector(3 downto 0);
	signal address: std_logic_vector(14 downto 0);
	signal reg: std_logic;
	
	component ram4bits is
	PORT
	(
		address		: IN STD_LOGIC_VECTOR (14 DOWNTO 0);
		clock		: IN STD_LOGIC  := '1';
		data		: IN STD_LOGIC_VECTOR (3 DOWNTO 0);
		wren		: IN STD_LOGIC ;
		q		: OUT STD_LOGIC_VECTOR (3 DOWNTO 0)
	);
	end component;
begin	
	process(clk,reset,valid,readEn)
	begin
		if(reset = '1') then
			address <= "000000000000000";
			reg <= '0';
		elsif(clk'event and clk = '1') then
			reg <= valid; --register the valid signal, don't know if this is necessary, we could just use the valid signal probably
			if(valid = '0' and readEn ='0') then
				address <= rwaddressstart;
			elsif(valid = '1' or readEn = '1') then -- if it's not a rising edge, just start adding 1 to the address
				address <= address + '1';
			end if;
		end if;
	end process;
	
	store: ram4bits port map(address => address,clock => clk,data => input,wren => reg,q=>hold);
	
	output <= hold when readEn = '1' else "0000"; --self explanatory
	
	--test signals
	addressout <= address;
	holdout <= hold;
	
end structure;